diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 3c73b0c..5592e1e 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -86,7 +86,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// assert!(t.is_empty()); /// ``` @@ -100,7 +100,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let _u = t.clone(); /// ``` @@ -111,7 +111,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// s(&t); /// ``` @@ -317,7 +317,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -335,7 +335,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -348,7 +348,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 2604c6a..a2f94ae 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -43,7 +43,10 @@ pub enum Source { translator: String, edition_year: u16, }, - /// Labelled model output; never in the test split, never judged by its own family. + /// Labelled model output; never in the test split. The rule that it is + /// never judged by its own family binds model judges and the split; a + /// deterministic scorer refuses nothing, so the loop labels (the round + /// counts items of the answerer's family) and does not forbid. /// `family` is `Unrecorded` when nothing records one. `generator` is the /// string the fixture or run wrote, kept verbatim. ModelWritten { @@ -54,7 +57,7 @@ pub enum Source { } /// The family of the model that wrote a line. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum ModelFamily { Anthropic, OpenAi, @@ -63,6 +66,7 @@ pub enum ModelFamily { Meta, Mistral, /// Nothing records which family wrote the line. + #[default] Unrecorded, } diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 5071316..a71a0b9 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -47,6 +47,73 @@ pub struct PacketDraft { pub struct DraftError { pub line: usize, pub column: usize, + /// What kind of reading failure, from serde's own fixed vocabulary. It + /// never holds a token of the answer. + pub class: DraftClass, +} + +/// Why an answer is not a draft: a closed set of classes, never the offending +/// text, so a record may keep it for a held-out item. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DraftClass { + /// Not JSON. + Syntax, + /// JSON that ends early. + Eof, + MissingField, + UnknownField, + /// A value of the wrong JSON type, such as a bare integer for an id. + InvalidType, + /// A value of the right type that is not one of the field's, such as a + /// number out of range. + InvalidValue, + /// A string that is not the form its field names: a node id spelled + /// `entity:x`, an id of the wrong kind ([`crate::form::FormError`]). + BadForm, + Other, +} + +impl DraftClass { + /// The code a round record keeps. + pub fn code(self) -> &'static str { + match self { + DraftClass::Syntax => "syntax", + DraftClass::Eof => "eof", + DraftClass::MissingField => "missing-field", + DraftClass::UnknownField => "unknown-field", + DraftClass::InvalidType => "invalid-type", + DraftClass::InvalidValue => "invalid-value", + DraftClass::BadForm => "bad-form", + DraftClass::Other => "other", + } + } + + fn of(e: &serde_json::Error) -> Self { + use serde_json::error::Category; + match e.classify() { + Category::Syntax => DraftClass::Syntax, + Category::Eof => DraftClass::Eof, + Category::Io => DraftClass::Other, + // Only the leading words of serde's message are read, and only to + // pick one of these classes; the message is not kept. + Category::Data => { + let msg = e.to_string(); + if msg.starts_with("missing field") { + DraftClass::MissingField + } else if msg.starts_with("unknown field") { + DraftClass::UnknownField + } else if msg.starts_with("invalid type") { + DraftClass::InvalidType + } else if msg.starts_with("invalid value") { + DraftClass::InvalidValue + } else if msg.starts_with("not a ") { + DraftClass::BadForm + } else { + DraftClass::Other + } + } + } + } } impl fmt::Display for DraftError { @@ -72,6 +139,7 @@ fn from_json(answer: &str) -> Result { serde_json::from_str(answer.trim()).map_err(|e| DraftError { line: e.line(), column: e.column(), + class: DraftClass::of(&e), }) } @@ -250,6 +318,31 @@ mod tests { CorpusItemId::parse("item:daodejing:1").unwrap() } + #[test] + fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() { + let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class; + assert_eq!(class("not json"), DraftClass::Syntax); + assert_eq!(class("{\"referents\": ["), DraftClass::Eof); + assert_eq!(class("{}"), DraftClass::MissingField); + // The classes serde's own messages open with, read off a plain error. + let of = |r: Result| DraftClass::of(&r.unwrap_err()); + assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue); + assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType); + // Every code is distinct and none can hold answer text: they are fixed. + let all = [ + DraftClass::Syntax, + DraftClass::Eof, + DraftClass::MissingField, + DraftClass::UnknownField, + DraftClass::InvalidType, + DraftClass::InvalidValue, + DraftClass::BadForm, + DraftClass::Other, + ]; + let codes: std::collections::BTreeSet<_> = all.iter().map(|c| c.code()).collect(); + assert_eq!(codes.len(), all.len()); + } + #[test] fn ids_are_pinned_digests_of_their_inputs() { let encoder = encoder_id("fake-model", "abcd"); diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 3808672..5b88c14 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -92,6 +92,10 @@ enum LoopCommand { /// rounds only) or tool-less-relay. Pinned in the run's record. #[arg(long)] lowerer_reach: LowererReach, + /// The family of the answering model; rounds count the corpus items + /// that family wrote. Defaults to unrecorded. + #[arg(long, default_value = "unrecorded")] + answerer_family: ModelFamilyArg, }, /// Run one round of the run at ROOT. The stores are read only from the /// run's record: the global --corpus, --inventory and --entities are not @@ -744,6 +748,7 @@ fn main() -> Result<()> { answerer, embed_model, lowerer_reach, + answerer_family, }) => { let run = RunStores::create( &root, @@ -756,6 +761,7 @@ fn main() -> Result<()> { &answerer, &embed_model, lowerer_reach, + answerer_family.into(), )?; println!("opened run:{}", run.record().label); } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index bcf0d00..6d0195f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -35,7 +35,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; -use crate::corpus::{CorpusItem, CorpusItemId, Source}; +use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; @@ -107,6 +107,12 @@ pub struct RunRecord { /// How the run's lowering prompts are answered, set once at open. #[cultcache(key = 12)] pub lowerer_reach: LowererReach, + /// The family of the model answering the run's encode and lowering + /// prompts, set once at open; `Unrecorded` when the opener did not say + /// and for a run opened before this was kept. + #[cultcache(key = 13, default)] + #[schemars(default)] + pub answerer_family: ModelFamily, } /// How a run's lowering prompts reach the model that answers them (rulings @@ -224,6 +230,8 @@ pub struct RefusalCounts { /// inventory_digest: String::new(), /// cycle: None, /// admit_rules: vec![], +/// parse_reasons: vec![], +/// own_family: 0, /// } /// } /// ``` @@ -276,9 +284,22 @@ pub struct RoundRecord { cycle: Option, /// The rules admission refused items under, in rule order; every Train /// and Validation item refused at `Stage::Admit` is counted once per rule. - #[cultcache(key = 18)] + #[cultcache(key = 18, default)] #[schemars(default)] admit_rules: Vec, + /// Why items were refused at `Stage::Parse`, in code order, over Train and + /// Validation: a [`DraftClass`] code (`rule`), or `unreadable` for an + /// item with no answer that read as text. Never a token of an answer. + #[cultcache(key = 19, default)] + #[schemars(default)] + parse_reasons: Vec, + /// Offered (live Train and Validation) `ModelWritten` items whose writer + /// family is the run's answerer family. The corpus rule binds + /// model judges; the scorer is deterministic, so nothing is refused for + /// this: the round labels it. + #[cultcache(key = 20, default)] + #[schemars(default)] + own_family: u32, } impl RoundRecord { @@ -336,6 +357,12 @@ impl RoundRecord { pub fn admit_rules(&self) -> &[RuleCount] { &self.admit_rules } + pub fn parse_reasons(&self) -> &[RuleCount] { + &self.parse_reasons + } + pub fn own_family(&self) -> u32 { + self.own_family + } /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -369,16 +396,24 @@ impl RoundRecord { x.admit, x.lower, ); - if self.admit_rules.is_empty() { - line - } else { - let rules: Vec = self - .admit_rules + let codes = |counts: &[RuleCount]| { + counts .iter() .map(|r| format!("{}x{}", r.rule, r.count)) - .collect(); - format!("{line} admit-rules={}", rules.join(",")) + .collect::>() + .join(",") + }; + let mut line = line; + if !self.parse_reasons.is_empty() { + line = format!("{line} parse-reasons={}", codes(&self.parse_reasons)); } + if !self.admit_rules.is_empty() { + line = format!("{line} admit-rules={}", codes(&self.admit_rules)); + } + if self.own_family > 0 { + line = format!("{line} own-family {}", self.own_family); + } + line } } @@ -550,13 +585,15 @@ impl RunStores { model: &str, embedder: &str, reach: LowererReach, + answerer_family: ModelFamily, ) -> Result { - Self::create_between(root, label, paths, model, embedder, reach, || {}) + Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {}) } /// `create` with `between` run after the digests are taken and before the /// stores are opened: the seam that lets a test change a store in that window. #[doc(hidden)] + #[allow(clippy::too_many_arguments)] // `create`'s six plus the test seam pub fn create_between( root: &Path, label: &str, @@ -564,6 +601,7 @@ impl RunStores { model: &str, embedder: &str, reach: LowererReach, + answerer_family: ModelFamily, between: impl FnOnce(), ) -> Result { if !is_slug(label) { @@ -592,6 +630,7 @@ impl RunStores { chrf_version: CHRF_VERSION.to_string(), inventory_digest: String::new(), lowerer_reach: reach, + answerer_family, }; // The run works on its own copy; the caller's store is never opened again. let head = root.join(INVENTORY_FILE); @@ -1024,13 +1063,21 @@ fn score_round( let mut items: Vec = Vec::new(); let mut fates: Vec = Vec::new(); let mut admit_rules: BTreeMap = BTreeMap::new(); + let mut parse_reasons: BTreeMap = BTreeMap::new(); // Phase 1 ends at admission: each admitted view waits for its lowering. let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new(); for (key, answer) in &answers { let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?; let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?; let fate = match answer.as_deref().map(PacketDraft::parse) { - None | Some(Err(_)) => Fate::Refused(Stage::Parse, None), + None => { + *parse_reasons.entry("unreadable".to_string()).or_default() += 1; + Fate::Refused(Stage::Parse, None) + } + Some(Err(e)) => { + *parse_reasons.entry(e.class.code().to_string()).or_default() += 1; + Fate::Refused(Stage::Parse, None) + } Some(Ok(draft)) => { let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); @@ -1172,6 +1219,15 @@ fn score_round( } } + let own_family = match run.record.answerer_family { + ModelFamily::Unrecorded => 0, + family => items + .iter() + .filter(|item| { + matches!(&item.source, Source::ModelWritten { family: f, .. } if *f == family) + }) + .count() as u32, + }; let n = run.rounds().map_err(store)?.len() as u32 + 1; let record = RoundRecord { n, @@ -1195,6 +1251,11 @@ fn score_round( .into_iter() .map(|(rule, count)| RuleCount { rule, count }) .collect(), + parse_reasons: parse_reasons + .into_iter() + .map(|(rule, count)| RuleCount { rule, count }) + .collect(), + own_family, }; Ok(RoundOutcome::Recorded(record)) } @@ -1628,6 +1689,8 @@ mod tests { inventory_digest: String::new(), cycle: None, admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, }; assert_eq!(r.n(), 1); } @@ -1844,6 +1907,8 @@ mod tests { inventory_digest: digest.to_string(), cycle: None, admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, } } @@ -1861,6 +1926,7 @@ mod tests { chrf_version: String::new(), inventory_digest: digest.to_string(), lowerer_reach: LowererReach::ToolLessRelay, + answerer_family: ModelFamily::Unrecorded, } } @@ -2059,7 +2125,7 @@ mod tests { entities: &entities, }, "fake-model", - "bag", crate::run::LowererReach::ToolLessRelay, + "bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); let mut model = fake(); diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index be64338..56a32af 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1346,6 +1346,8 @@ fn loop_open_and_unanswered_round() { "fake-model", "--lowerer-reach", "tool-less-relay", + "--answerer-family", + "open-ai", ], ) }; @@ -1395,7 +1397,7 @@ fn loop_open_and_unanswered_round() { assert!(second.status.success(), "{}", out(&second)); assert_eq!( out(&second), - "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 lowering=tool-less-relay\n" + "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" ); // The global store flags are not read by loop round: a drifted store is refused. diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 189202b..7bd9338 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -131,7 +131,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { entities: &dir.join("entities.cc"), }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, ) .unwrap() } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 4d9dc1c..60abc38 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use common::{ent, run_over, world}; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; +use weksa::corpus::ModelFamily; use weksa::run::{DriftOf, RunError, RunPaths, RunRecord, RunStores}; use weksa::split::fnv1a64; @@ -47,7 +48,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { entities: &e, }, model, - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; @@ -148,7 +149,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { entities: &e, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); @@ -223,7 +224,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; @@ -268,7 +269,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; // A directory opens for reading and fails mid-copy, after the destination exists. @@ -381,14 +382,14 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { entities: &e, }; assert_eq!( - RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || more(&mut w)).err(), + RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(), Some(RunError::Drift(DriftOf::Corpus)) ); assert!(!root.join("inventory.cc").exists()); // Not the copy and not the lock the store's open left beside it. assert!(!root.join("inventory.cc.lock").exists()); assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun)); - RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay).unwrap(); + RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap(); } /// Runs `change` in the window between the pins and the open, at `open` and at @@ -411,7 +412,7 @@ fn window( inventory: &i, entities: &e, }; - let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || { + let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || { change(&mut w, &root) }) .err(); @@ -828,7 +829,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao { entities: &entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); Dao { @@ -1477,15 +1478,15 @@ fn errors_echo_no_input() { inventory: &missing, entities: &missing, }; - let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); - let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); - let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); @@ -1519,7 +1520,7 @@ fn errors_echo_no_input() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); let mut model = cycle_fake(&format!("{{\"{canary}\":1}}")); @@ -2449,7 +2450,7 @@ fn a_changed_head_is_drift() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); assert_eq!( @@ -2768,7 +2769,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { inventory: &dao.inventory, entities: &dao.entities, }; - RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent) + RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded) .unwrap(); let run = RunStores::open(&root).unwrap(); assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent); @@ -2947,6 +2948,282 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { assert!(!dao.run.round_line(&r).contains("admit-rules")); } +/// Item 1's good draft as JSON, changed by `change`, as the model's answer. +fn mangled(change: impl FnOnce(&mut serde_json::Value)) -> String { + let mut v: serde_json::Value = serde_json::from_str(&json(&draft_of(1))).unwrap(); + change(&mut v); + v.to_string() +} + +/// The round record of a Dao run whose item 1 is answered with `answer` and +/// the rest well. +fn round_with_item_1(answer: String) -> (Dao, RoundRecord) { + let mut dao = dao(); + let mut answers = good_answers(); + answers[0].1 = answer; + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + (dao, r) +} + +fn reasons(r: &RoundRecord) -> Vec<(&str, u32)> { + r.parse_reasons() + .iter() + .map(|c| (c.rule.as_str(), c.count)) + .collect() +} + +#[test] +fn parse_reasons_by_class() { + let good = json(&draft_of(1)); + let truncated = good[..good.len() / 2].to_string(); + let cases: Vec<(&str, String)> = vec![ + ("syntax", "not json at all".to_string()), + ("eof", truncated), + ( + "invalid-type", + mangled(|v| v["referents"][0]["concept"] = serde_json::json!(7)), + ), + ( + "bad-form", + mangled(|v| v["referents"][0]["concept"] = serde_json::json!("entity:x")), + ), + ( + "unknown-field", + mangled(|v| v["surprise"] = serde_json::json!(1)), + ), + ( + "missing-field", + mangled(|v| { + v.as_object_mut().unwrap().remove("referents"); + }), + ), + ]; + for (code, answer) in cases { + let (dao, r) = round_with_item_1(answer); + assert_eq!(reasons(&r), [(code, 1)], "{code}"); + assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}"); + let line = dao.run.round_line(&r); + assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}"); + } + // A round nothing refused at Parse carries and prints none. + let (dao, r) = round_with_item_1(good); + assert!(r.parse_reasons().is_empty()); + assert!(!dao.run.round_line(&r).contains("parse-reasons")); +} + +#[test] +fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { + let mut dao = dao(); + let expected = prompts(&dao.run); + let answers = keyed(&good_answers()); + let round1 = dao.root.join("round-1"); + std::fs::create_dir_all(&round1).unwrap(); + for (key, p) in &expected { + let bytes = match key.as_str() { + // Train: not UTF-8, so no answer parsed from the file. + "item:daodejing:1" => vec![0xff, 0xfe], + // Validation: not JSON. + "item:daodejing:2" => b"nope".to_vec(), + _ => answers[key].clone().into_bytes(), + }; + std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap(); + } + let table = table_english(); + for (key, p) in lowering_prompts(&dao, dao.run.inventory()) + .iter() + .filter(|(key, _)| key == "item:daodejing:3") + { + let answer = serde_json::json!({ "text": table[key] }).to_string(); + std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); + } + let mut files = AnswerFiles::new(round1, "fake-model".into()); + let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); + assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); +} + +#[test] +fn parse_reasons_never_echo_answer() { + let canary = "CANARY-7f3a-held-out"; + // The canary sits where serde would quote it: an unknown field name, an + // invalid value, and the raw text of a non-JSON answer. + let answers = [ + mangled(|v| v[canary] = serde_json::json!(1)), + mangled(|v| v["referents"][0]["concept"] = serde_json::json!(canary)), + format!("{canary} is not json"), + ]; + for answer in answers { + // Item 2 is Validation: the held-out item carries the canary. + let mut dao = dao(); + let mut a = good_answers(); + a[1].1 = answer; + let mut model = fake(&a); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(r.validation_refused().parse, 1); + let line = dao.run.round_line(&r); + assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}"); + let bytes = std::fs::read(dao.root.join("run.cc")).unwrap(); + assert!(!bytes.windows(6).any(|w| w == b"CANARY")); + assert!(!format!("{r:?}").contains("CANARY")); + } +} + +/// A run record as written before `answerer_family` (key 13) existed. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_run")] +struct PreFamilyRun { + #[cultcache(key = 0)] + label: String, + #[cultcache(key = 1)] + corpus: String, + #[cultcache(key = 2)] + inventory: String, + #[cultcache(key = 3)] + entities: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + prompt_digest: String, + #[cultcache(key = 8)] + model: String, + #[cultcache(key = 9)] + embedder: String, + #[cultcache(key = 10)] + chrf_version: String, + #[cultcache(key = 11)] + inventory_digest: String, + #[cultcache(key = 12)] + lowerer_reach: weksa::run::LowererReach, +} + +/// A round record as written before `parse_reasons` and `own_family`, and +/// before `admit_rules` had a default. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct LegacyRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 11)] + train_mean: weksa::score::SplitMean, + #[cultcache(key = 12)] + validation_mean: weksa::score::SplitMean, + #[cultcache(key = 13)] + validation_refused: weksa::run::RefusalCounts, + #[cultcache(key = 14)] + train_scored: u32, + #[cultcache(key = 15)] + validation_scored: u32, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, +} + +#[test] +fn records_from_before_the_new_fields_decode_with_defaults() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rec = dao.run.record().clone(); + let root = dao.root.clone(); + let legacy = LegacyRound { + n: r.n(), + inventory_rev: r.inventory_rev(), + prompt_digest: r.prompt_digest().into(), + corpus_digest: r.corpus_digest().into(), + entity_digest: r.entity_digest().into(), + model: r.model().into(), + embedder: r.embedder().into(), + chrf_version: r.chrf_version().into(), + calls: r.calls().to_vec(), + train: r.train().to_vec(), + train_mean: r.train_mean().clone(), + validation_mean: r.validation_mean().clone(), + validation_refused: r.validation_refused(), + train_scored: r.train_scored(), + validation_scored: r.validation_scored(), + inventory_digest: r.inventory_digest().into(), + cycle: r.cycle(), + }; + let run = PreFamilyRun { + label: rec.label.clone(), + corpus: rec.corpus.clone(), + inventory: rec.inventory.clone(), + entities: rec.entities.clone(), + corpus_digest: rec.corpus_digest.clone(), + entity_digest: rec.entity_digest.clone(), + prompt_digest: rec.prompt_digest.clone(), + model: rec.model.clone(), + embedder: rec.embedder.clone(), + chrf_version: rec.chrf_version.clone(), + inventory_digest: rec.inventory_digest.clone(), + lowerer_reach: rec.lowerer_reach, + }; + drop(dao.run); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &run).unwrap(); + cache.put("run:dao:round:1".to_string(), &legacy).unwrap(); + drop(cache); + + let reopened = RunStores::open(&root).unwrap(); + assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded); + let rounds = reopened.rounds().unwrap(); + assert_eq!(rounds.len(), 1); + assert_eq!(rounds[0].n(), 1); + assert!(rounds[0].admit_rules().is_empty()); + assert!(rounds[0].parse_reasons().is_empty()); + assert_eq!(rounds[0].own_family(), 0); +} + +#[test] +fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { + let published = weksa::schema::published(); + let schema = |needle: &str| -> serde_json::Value { + let (_, json) = published + .iter() + .find(|(name, _)| name.contains(needle)) + .unwrap(); + serde_json::from_str(json).unwrap() + }; + for (needle, fields) in [ + ("loop_round", &["parse_reasons", "own_family"][..]), + ("loop_run", &["answerer_family"][..]), + ] { + let s = schema(needle); + let required = s["required"].as_array().unwrap(); + for f in fields { + assert!(s["properties"][f].is_object(), "{needle} {f}"); + assert!(!required.iter().any(|r| r == f), "{needle} {f} is required"); + } + } +} + /// Answers a recall probe from the corpus: the item's own second half, a /// fixed wrong text, or junk. struct Reciter { diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 51e4c35..cffd4f2 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -99,7 +99,7 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { }, "m", "bag", - weksa::run::LowererReach::BriefedSubagent, + weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded ) .unwrap(); let mut model = Unrelated(vec![]); @@ -112,3 +112,79 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { assert!(model.0.iter().all(|t| t.contains("work: not published"))); assert!(model.0.iter().all(|t| !t.contains("translator:"))); } + +/// Answers every prompt with text that is not a draft: nothing is admitted, so +/// the round measures only what is counted before admission. +struct Nope; + +impl weksa::model::ModelPort for Nope { + fn model(&self) -> &str { + "nope" + } + fn answer( + &mut self, + _: &weksa::context::Prompt, + ) -> Result { + Ok("not a draft".to_string()) + } +} + +struct Bag; + +impl weksa::score::Embedder for Bag { + fn model(&self) -> &str { + "bag" + } + fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts.iter().map(|_| vec![1.0]).collect()) + } +} + +fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::RoundRecord) { + use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap().keep(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(src.join(f), tmp.join(f)).unwrap(); + } + let mut run = RunStores::create( + &tmp.join("run"), + "orig", + RunPaths { + corpus: &tmp.join("corpus.cc"), + inventory: &tmp.join("inventory.cc"), + entities: &tmp.join("entity.cc"), + }, + "nope", + "bag", + LowererReach::ToolLessRelay, + family, + ) + .unwrap(); + let scorer = weksa::score::Scorer::new(Bag); + match round(&mut run, &mut Nope, &scorer).unwrap() { + RoundOutcome::Recorded(r) => (run, r), + RoundOutcome::Unanswered { .. } => panic!("answered"), + } +} + +#[test] +fn own_family_counts_and_labels() { + // The original corpus is six Anthropic-written items, all offered. + let (run, r) = original_round(ModelFamily::Anthropic); + assert_eq!(r.own_family(), 6); + assert_eq!( + r.parse_reasons().iter().map(|c| c.count).sum::(), + 6, + "every item was refused at Parse and says why" + ); + let line = run.round_line(&r); + assert!(line.contains(" own-family 6"), "{line}"); + + // A different answerer family, or none recorded, labels nothing. + for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { + let (run, r) = original_round(family); + assert_eq!(r.own_family(), 0); + assert!(!run.round_line(&r).contains("own-family")); + } +} diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index c82c4da..b5e5bc9 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -218,7 +218,7 @@ ] }, { - "description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.", + "description": "Labelled model output; never in the test split. The rule that it is\nnever judged by its own family binds model judges and the split; a\ndeterministic scorer refuses nothing, so the loop labels (the round\ncounts items of the answerer's family) and does not forbid.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.", "type": "object", "properties": { "ModelWritten": { diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index f218bdb..4117573 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n }\n}\n```", "type": "object", "properties": { "admit_rules": { @@ -57,6 +57,21 @@ "format": "uint32", "minimum": 0 }, + "own_family": { + "description": "Offered (live Train and Validation) `ModelWritten` items whose writer\nfamily is the run's answerer family. The corpus rule binds\nmodel judges; the scorer is deterministic, so nothing is refused for\nthis: the round labels it.", + "type": "integer", + "format": "uint32", + "default": 0, + "minimum": 0 + }, + "parse_reasons": { + "description": "Why items were refused at `Stage::Parse`, in code order, over Train and\nValidation: a [`DraftClass`] code (`rule`), or `unreadable` for an\nitem with no answer that read as text. Never a token of an answer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/RuleCount" + } + }, "prompt_digest": { "type": "string" }, diff --git a/schemas/cultnet/weksa.loop_run.schema.json b/schemas/cultnet/weksa.loop_run.schema.json index b5b5ab8..b339def 100644 --- a/schemas/cultnet/weksa.loop_run.schema.json +++ b/schemas/cultnet/weksa.loop_run.schema.json @@ -4,6 +4,11 @@ "description": "What a run pinned at open. Key `run: