diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 3d41aa7..299ea72 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -23,6 +23,7 @@ //! uses the template table. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::io::ErrorKind; use std::path::{Path, PathBuf}; @@ -40,7 +41,9 @@ use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; use crate::score::{CHRF_VERSION, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; -use crate::store::{CorpusStore, EntityStore, InventoryStore, PacketStore, open_cache_over}; +use crate::store::{ + CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over, +}; use crate::validate::LoweringInput; /// The digest of the prompt builder's source: `context.rs`, then `split.rs`, @@ -164,6 +167,14 @@ pub struct Call { pub items: Vec, } +/// How many items one admission rule refused, over both splits: a rule code +/// (`V5`, `UnknownEntity`) and a count, never an identifier or text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct RuleCount { + pub rule: String, + pub count: u32, +} + /// A Train item's row. Validation has no rows. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct TrainRow { @@ -212,6 +223,7 @@ pub struct RefusalCounts { /// validation_refused: RefusalCounts::default(), /// inventory_digest: String::new(), /// cycle: None, +/// admit_rules: vec![], /// } /// } /// ``` @@ -262,6 +274,10 @@ pub struct RoundRecord { /// `Some(k)` for the candidate round of cycle `k`. #[cultcache(key = 17)] cycle: Option, + /// The rules admission refused items under, in rule order; every Train + /// and Validation item refused at `Stage::Admit` is counted once per rule. + #[cultcache(key = 18)] + admit_rules: Vec, } impl RoundRecord { @@ -316,6 +332,9 @@ impl RoundRecord { pub fn cycle(&self) -> Option { self.cycle } + pub fn admit_rules(&self) -> &[RuleCount] { + &self.admit_rules + } /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -333,7 +352,7 @@ impl RoundRecord { &self.validation_mean, self.validation_refused, ); - format!( + let line = format!( "round {} rev {} train n={} scored={} chrf={} meaning={} refused={refused} validation n={} scored={} chrf={} meaning={} refused parse={} admit={} lower={}", self.n, self.inventory_rev, @@ -348,7 +367,17 @@ impl RoundRecord { x.parse, x.admit, x.lower, - ) + ); + if self.admit_rules.is_empty() { + line + } else { + let rules: Vec = self + .admit_rules + .iter() + .map(|r| format!("{}x{}", r.rule, r.count)) + .collect(); + format!("{line} admit-rules={}", rules.join(",")) + } } } @@ -772,6 +801,23 @@ pub enum RoundOutcome { Recorded(RoundRecord), } +/// The distinct rule codes an admission refusal broke: validator rules by +/// code, a store refusal by its variant name. Nothing from the packet or the +/// stores is kept; an I/O failure is `io`. +fn admit_rule_codes(e: &StoreError) -> BTreeSet { + match e { + StoreError::Invalid(rs) => rs.iter().map(|r| r.rule().to_string()).collect(), + StoreError::Refused(r) => [r + .to_string() + .split([' ', ':']) + .next() + .unwrap_or_default() + .to_string()] + .into(), + StoreError::Io(_) => ["io".to_string()].into(), + } +} + /// How one item fared: where it was refused, or the text it lowered to, with /// the key of the packet that was admitted for it. enum Fate { @@ -876,6 +922,7 @@ fn score_round( let encoder = encoder_id(model.model(), &run.record.prompt_digest); let mut items: Vec = Vec::new(); let mut fates: Vec = Vec::new(); + let mut admit_rules: BTreeMap = BTreeMap::new(); // Phase 1 ends at admission: each admitted view waits for its lowering. let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new(); for (key, answer) in &answers { @@ -887,7 +934,12 @@ fn score_round( let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); match packets.admit(packet, inventory_store, &run.corpus, &run.entities) { - Err(_) => Fate::Refused(Stage::Admit, None), + Err(e) => { + for rule in admit_rule_codes(&e) { + *admit_rules.entry(rule).or_default() += 1; + } + Fate::Refused(Stage::Admit, None) + } Ok(view) => { admitted.push((fates.len(), id, packet_key, view)); // Replaced below by what the lowering answers. @@ -1038,6 +1090,10 @@ fn score_round( validation_scored, inventory_digest: inventory_digest.to_string(), cycle, + admit_rules: admit_rules + .into_iter() + .map(|(rule, count)| RuleCount { rule, count }) + .collect(), }; Ok(RoundOutcome::Recorded(record)) } @@ -1470,6 +1526,7 @@ mod tests { validation_refused: RefusalCounts::default(), inventory_digest: String::new(), cycle: None, + admit_rules: vec![], }; assert_eq!(r.n(), 1); } @@ -1685,6 +1742,7 @@ mod tests { validation_refused: RefusalCounts::default(), inventory_digest: digest.to_string(), cycle: None, + admit_rules: vec![], } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index a4bba52..6269f99 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2898,3 +2898,51 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { unique.dedup(); assert_eq!(unique.len(), 2); } + +/// Several refusals under one rule in one packet: the item still counts once. +fn every_concept_unknown(d: &mut PacketDraft) { + for r in &mut d.referents { + r.concept = "concept:999".parse().unwrap(); + } + for p in &mut d.predications { + p.concept = "concept:999".parse().unwrap(); + } +} + +#[test] +fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { + use weksa::packet::EntityId; + let mut dao = dao(); + let empty = |d: &mut PacketDraft| { + d.referents.clear(); + d.predications.clear(); + d.relations.clear(); + }; + let nobody = |d: &mut PacketDraft| { + d.referents[0].entity = Some("entity:nobody".parse::().unwrap()); + }; + let mut model = fake(&[ + (1, edited(1, nobody)), + (2, edited(2, every_concept_unknown)), + (3, edited(3, empty)), + ]); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rules: Vec<(&str, u32)> = r + .admit_rules() + .iter() + .map(|c| (c.rule.as_str(), c.count)) + .collect(); + assert_eq!(rules, [("UnknownEntity", 1), ("V2", 1), ("V5", 1)]); + let line = dao.run.round_line(&r); + assert!( + line.ends_with("admit-rules=UnknownEntityx1,V2x1,V5x1 lowering=tool-less-relay"), + "{line}" + ); + + // A round nothing refused names no rule and prints none. + let mut dao = self::dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert!(r.admit_rules().is_empty()); + assert!(!dao.run.round_line(&r).contains("admit-rules")); +} diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index 27205d1..dd940ad 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,9 +1,16 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```", "type": "object", "properties": { + "admit_rules": { + "description": "The rules admission refused items under, in rule order; every Train\nand Validation item refused at `Stage::Admit` is counted once per rule.", + "type": "array", + "items": { + "$ref": "#/$defs/RuleCount" + } + }, "calls": { "description": "The encode calls, then the lowering calls, each with its prompt digest\nand one item key.", "type": "array", @@ -96,7 +103,8 @@ "validation_refused", "train_scored", "validation_scored", - "inventory_digest" + "inventory_digest", + "admit_rules" ], "$defs": { "Call": { @@ -187,6 +195,24 @@ "lower" ] }, + "RuleCount": { + "description": "How many items one admission rule refused, over both splits: a rule code\n(`V5`, `UnknownEntity`) and a count, never an identifier or text.", + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "rule": { + "type": "string" + } + }, + "required": [ + "rule", + "count" + ] + }, "SplitMean": { "description": "Mean scores of one derived split.", "type": "object",