From 1a20c977e0ff3f31416a21a3ad42d926ff152343 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 17:31:56 +0000 Subject: [PATCH 1/4] Pin own_family to offered items and loop open --answerer-family with tests Mixed-family corpus fixture (Anthropic, OpenAi, Unrecorded, PublicDomain, cast, a Test-group model item, a withdrawn item) and a CLI test reading the run record after `loop open`. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/cli.rs | 39 +++++++++++ crates/weksa/tests/golf_loop.rs | 119 ++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 56a32af..4113a82 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1431,6 +1431,45 @@ fn open_dao_run(d: &std::path::Path) { assert!(o.status.success(), "{}", out(&o)); } +/// The family `loop open` pinned at `root`, read from the run record. +fn pinned_family(d: &std::path::Path, root: &str, flag: &[&str]) -> weksa::corpus::ModelFamily { + let mut args = vec![ + "loop", + "open", + root, + "--label", + "fam", + "--answerer", + "fake-model", + "--lowerer-reach", + "tool-less-relay", + ]; + args.extend_from_slice(flag); + let o = weksa(d, &args); + assert!(o.status.success(), "{}", out(&o)); + weksa::run::RunStores::open(&d.join(root)) + .unwrap() + .record() + .answerer_family +} + +#[test] +fn loop_open_pins_answerer_family() { + use weksa::corpus::ModelFamily; + let d = tempfile::tempdir().unwrap(); + dao_stores(d.path()); + assert_eq!( + pinned_family(d.path(), "a", &["--answerer-family", "anthropic"]), + ModelFamily::Anthropic + ); + assert_eq!( + pinned_family(d.path(), "b", &["--answerer-family", "mistral"]), + ModelFamily::Mistral + ); + // Omitted, the family stays unrecorded and a round prints no own-family line. + assert_eq!(pinned_family(d.path(), "c", &[]), ModelFamily::Unrecorded); +} + #[test] fn loop_cycle_needs_a_floor() { let d = tempfile::tempdir().unwrap(); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 60abc38..9d6d1ed 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -3365,3 +3365,122 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdra let unnamed = m.seen.iter().filter(|t| t.contains("work: not published")).count(); assert_eq!((named, unnamed), (2, 1)); } + +/// Answers every prompt with text that is not a draft, so a round measures only +/// what it counts before admission. +struct NotADraft; + +impl weksa::model::ModelPort for NotADraft { + fn model(&self) -> &str { + "not-a-draft" + } + fn answer(&mut self, _: &weksa::context::Prompt) -> Result { + Ok("not a draft".to_string()) + } +} + +struct OneBag; + +impl weksa::score::Embedder for OneBag { + fn model(&self) -> &str { + "bag" + } + fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts.iter().map(|_| vec![1.0]).collect()) + } +} + +/// A corpus of every shape a family count must tell apart, and the own-family +/// count of a round over it by an answerer of `family`. +fn mixed_family_round(family: ModelFamily) -> u32 { + use common::{cast, group_in_bucket, item_in}; + use weksa::corpus::Source; + use weksa::run::{LowererReach, RoundOutcome, round}; + + let mut w = world(); + let written = |family| Source::ModelWritten { + family, + generator: None, + prompt_digest: None, + }; + let public_domain = Source::PublicDomain { + work: "w".into(), + author: "a".into(), + edition_url: "https://example.org/e".into(), + licence: "PD".into(), + jurisdiction: "US".into(), + translator: "t".into(), + edition_year: 1900, + }; + let (train, validation, test) = ( + group_in_bucket(10), + group_in_bucket(50), + group_in_bucket(90), + ); + let items = [ + // Offered, Anthropic: two of them count for Anthropic. + ("item:mix:1", &train, written(ModelFamily::Anthropic)), + ("item:mix:2", &validation, written(ModelFamily::Anthropic)), + // Offered, other families or no model at all: never Anthropic's. + ("item:mix:3", &train, written(ModelFamily::OpenAi)), + ("item:mix:4", &train, written(ModelFamily::Unrecorded)), + ("item:mix:5", &train, public_domain), + ("item:mix:6", &train, cast()), + // Model-written in a Test group: in no split, never offered. + ("item:mix:7", &test, written(ModelFamily::Anthropic)), + // Withdrawn after admission: never offered. + ("item:mix:8", &train, written(ModelFamily::Anthropic)), + ]; + let entities = common::entities_in(w.dir.path()); + for (key, group, source) in items { + let text = format!("Line {key}, distinct."); + w.corpus + .add(item_in(key, &text, group, source), &entities) + .unwrap(); + } + w.corpus.withdraw("item:mix:8").unwrap(); + let dir = w.dir.path().to_path_buf(); + for name in ["c.cc", "i.cc", "entities.cc"] { + assert!(dir.join(name).exists(), "{name}"); + } + let mut run = RunStores::create( + &dir.join("run"), + "mix", + RunPaths { + corpus: &dir.join("c.cc"), + inventory: &dir.join("i.cc"), + entities: &dir.join("entities.cc"), + }, + "not-a-draft", + "bag", + LowererReach::ToolLessRelay, + family, + ) + .unwrap(); + let scorer = weksa::score::Scorer::new(OneBag); + match round(&mut run, &mut NotADraft, &scorer).unwrap() { + RoundOutcome::Recorded(r) => { + // Seven items were offered: the world's own cast item and mix 1 to 6; + // the Test-group and withdrawn items were not. + assert_eq!( + r.parse_reasons().iter().map(|c| c.count).sum::(), + 7, + "offered items" + ); + r.own_family() + } + RoundOutcome::Unanswered { .. } => panic!("answered"), + } +} + +#[test] +fn own_family_counts_offered_items_of_the_answerer_family() { + // Two live Train/Validation Anthropic items; the Test-group and withdrawn + // Anthropic items are not offered, so they are not counted. + assert_eq!(mixed_family_round(ModelFamily::Anthropic), 2); + assert_eq!(mixed_family_round(ModelFamily::OpenAi), 1); + // Google wrote nothing here; an unrecorded answerer counts no item, though + // an offered item is itself unrecorded. + assert_eq!(mixed_family_round(ModelFamily::Google), 0); + assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0); +} -- 2.47.3 From 99286d3892956089a5028d347b95e0f80f2c9d9d Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 17:33:53 +0000 Subject: [PATCH 2/4] DraftClass: split the data-message reader out and pin each class with a real serde error serde_json exposes no kind for a data error, so the prefix reader stays as DraftClass::of_data; the test feeds each class a real answer and an unknown message to Other. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 60 +++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index a71a0b9..3b9f437 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -94,24 +94,27 @@ impl DraftClass { Category::Syntax => DraftClass::Syntax, Category::Eof => DraftClass::Eof, Category::Io => DraftClass::Other, - // Only the leading words of serde's message are read, and only to - // pick one of these classes; the message is not kept. - Category::Data => { - let msg = e.to_string(); - if msg.starts_with("missing field") { - DraftClass::MissingField - } else if msg.starts_with("unknown field") { - DraftClass::UnknownField - } else if msg.starts_with("invalid type") { - DraftClass::InvalidType - } else if msg.starts_with("invalid value") { - DraftClass::InvalidValue - } else if msg.starts_with("not a ") { - DraftClass::BadForm - } else { - DraftClass::Other - } - } + Category::Data => Self::of_data(&e.to_string()), + } + } + + /// serde_json offers no kind for a data error: its message opens with serde's + /// own fixed words, or with a [`crate::form::FormError`]'s `not a `. Only + /// those words are read, to pick a class; the message is not kept, and any + /// other opening is [`DraftClass::Other`]. + fn of_data(msg: &str) -> Self { + if msg.starts_with("missing field") { + DraftClass::MissingField + } else if msg.starts_with("unknown field") { + DraftClass::UnknownField + } else if msg.starts_with("invalid type") { + DraftClass::InvalidType + } else if msg.starts_with("invalid value") { + DraftClass::InvalidValue + } else if msg.starts_with("not a ") { + DraftClass::BadForm + } else { + DraftClass::Other } } } @@ -319,15 +322,26 @@ mod tests { } #[test] - fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() { + fn draft_class_per_real_serde_error() { + // Each class is fed by the error serde_json raises for a real answer. let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class; assert_eq!(class("not json"), DraftClass::Syntax); assert_eq!(class("{\"referents\": ["), DraftClass::Eof); assert_eq!(class("{}"), DraftClass::MissingField); - // The classes serde's own messages open with, read off a plain error. - let of = |r: Result| DraftClass::of(&r.unwrap_err()); - assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue); - assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType); + assert_eq!(class("{\"bogus\": 1}"), DraftClass::UnknownField); + assert_eq!(class("{\"confidence\": \"x\"}"), DraftClass::InvalidType); + assert_eq!(class("{\"confidence\": -1}"), DraftClass::InvalidValue); + let bad_id = r#"{"referents": [{"id": "entity:x"}]}"#; + assert_eq!(class(bad_id), DraftClass::BadForm); + // A data error opening with words this reader does not know is Other. + assert_eq!( + DraftClass::of_data("something serde never said"), + DraftClass::Other + ); + assert_eq!(DraftClass::of_data(""), DraftClass::Other); + // None keeps a token of the answer. + let secret = r#"{"confidence": "canary-token"}"#; + assert!(!format!("{:?}", PacketDraft::parse(secret).unwrap_err()).contains("canary")); // Every code is distinct and none can hold answer text: they are fixed. let all = [ DraftClass::Syntax, -- 2.47.3 From 99cf0cb39fd730891224d40870cb77814c86de61 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 17:33:53 +0000 Subject: [PATCH 3/4] Remove rustfmt diffs the parse-reason cut added to run.rs and tests Only hunks on lines that cut changed; import ordering and older code untouched. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 15 +++- crates/weksa/tests/common/mod.rs | 4 +- crates/weksa/tests/golf_loop.rs | 124 +++++++++++++++++++++----- crates/weksa/tests/original_corpus.rs | 8 +- 4 files changed, 119 insertions(+), 32 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 6d0195f..943894d 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -587,7 +587,16 @@ impl RunStores { reach: LowererReach, answerer_family: ModelFamily, ) -> Result { - Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {}) + Self::create_between( + root, + label, + paths, + model, + embedder, + reach, + answerer_family, + || {}, + ) } /// `create` with `between` run after the digests are taken and before the @@ -2125,7 +2134,9 @@ mod tests { entities: &entities, }, "fake-model", - "bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + crate::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) .unwrap(); let mut model = fake(); diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 7bd9338..73c8456 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -131,7 +131,9 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { entities: &dir.join("entities.cc"), }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, + "bag", + weksa::run::LowererReach::ToolLessRelay, + weksa::corpus::ModelFamily::Unrecorded, ) .unwrap() } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 9d6d1ed..0929e53 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -48,7 +48,9 @@ fn a_run_pins_its_stores_and_refuses_drift() { entities: &e, }, model, - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) }; @@ -149,7 +151,9 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { entities: &e, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) .unwrap(); @@ -224,7 +228,9 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) }; @@ -269,7 +275,9 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) }; // A directory opens for reading and fails mid-copy, after the destination exists. @@ -382,14 +390,33 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { entities: &e, }; assert_eq!( - RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(), + RunStores::create_between( + &root, + "t", + paths(), + "fake-model", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + || more(&mut w) + ) + .err(), Some(RunError::Drift(DriftOf::Corpus)) ); assert!(!root.join("inventory.cc").exists()); // Not the copy and not the lock the store's open left beside it. assert!(!root.join("inventory.cc.lock").exists()); assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun)); - RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap(); + RunStores::create( + &root, + "t", + paths(), + "fake-model", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + ) + .unwrap(); } /// Runs `change` in the window between the pins and the open, at `open` and at @@ -412,9 +439,16 @@ fn window( inventory: &i, entities: &e, }; - let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || { - change(&mut w, &root) - }) + let created = RunStores::create_between( + &root, + "t", + paths, + "fake-model", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + || change(&mut w, &root), + ) .err(); let left = root.join("inventory.cc").exists() || root.join("inventory.cc.lock").exists(); (opened, created, left, RunStores::open(&root).err()) @@ -829,7 +863,9 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao { entities: &entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) .unwrap(); Dao { @@ -1478,17 +1514,41 @@ fn errors_echo_no_input() { inventory: &missing, entities: &missing, }; - let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) - .err() - .unwrap(); + let e = RunStores::create( + &root, + "ok", + stores, + "m", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + ) + .err() + .unwrap(); note(&e, &e); - let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) - .err() - .unwrap(); + let e = RunStores::create( + &root, + &format!("{canary}!"), + stores, + "m", + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + ) + .err() + .unwrap(); note(&e, &e); - let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) - .err() - .unwrap(); + let e = RunStores::create( + &root, + "ok", + stores, + &format!("{canary}\n"), + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, + ) + .err() + .unwrap(); note(&e, &e); let w = world(); let run = run_over(w.dir.path()); @@ -1520,7 +1580,9 @@ fn errors_echo_no_input() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) .unwrap(); let mut model = cycle_fake(&format!("{{\"{canary}\":1}}")); @@ -2450,7 +2512,9 @@ fn a_changed_head_is_drift() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded + "bag", + weksa::run::LowererReach::ToolLessRelay, + ModelFamily::Unrecorded, ) .unwrap(); assert_eq!( @@ -2769,8 +2833,16 @@ fn a_summary_line_says_how_lowerings_were_answered() { inventory: &dao.inventory, entities: &dao.entities, }; - RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded) - .unwrap(); + RunStores::create( + &root, + "b", + paths, + "fake-model", + "bag", + LowererReach::BriefedSubagent, + ModelFamily::Unrecorded, + ) + .unwrap(); let run = RunStores::open(&root).unwrap(); assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent); assert_eq!( @@ -3002,7 +3074,11 @@ fn parse_reasons_by_class() { for (code, answer) in cases { let (dao, r) = round_with_item_1(answer); assert_eq!(reasons(&r), [(code, 1)], "{code}"); - assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}"); + assert_eq!( + r.train()[0].outcome, + Outcome::Refused(Stage::Parse), + "{code}" + ); let line = dao.run.round_line(&r); assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}"); } diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index cffd4f2..91b635b 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -99,7 +99,8 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { }, "m", "bag", - weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded + weksa::run::LowererReach::BriefedSubagent, + ModelFamily::Unrecorded, ) .unwrap(); let mut model = Unrelated(vec![]); @@ -121,10 +122,7 @@ impl weksa::model::ModelPort for Nope { fn model(&self) -> &str { "nope" } - fn answer( - &mut self, - _: &weksa::context::Prompt, - ) -> Result { + fn answer(&mut self, _: &weksa::context::Prompt) -> Result { Ok("not a draft".to_string()) } } -- 2.47.3 From 7ee9d73c729f3e56cd38e480911153e34cc2b323 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 17:34:44 +0000 Subject: [PATCH 4/4] Encode prompt: a referent's concept is never null Both parse refusals in the original-corpus round (runs/orig-3) were a null referents[].concept; the prompt gave 'or null' to entity, setting, medium and addressee and never said concept could not be. One sentence, pinned by a test. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/tests/context.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 5592e1e..1a648a0 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -59,7 +59,7 @@ Fields: - predications: a list of {id, concept, frame, roles, features}; roles is a list of {role, filler}, filler a node id. - relations: a list of {relation, from, to, features}. - stance: {features}; the speaker's felt affect and restraint, never the affect the line displays. -Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. +Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A referent's concept is never null, even when it carries an entity: give the nearest concept id. A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": label} or {\"Flag\": true or false}, as the inventory lists the feature's kind."; /// The instructions a lowering prompt starts with: the lowerer's whole brief. diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index c379f0e..d6e2efe 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -694,6 +694,7 @@ mod lowering { #[test] fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { assert!(ENCODE.contains("family prefix included: concept:7, never 7")); + assert!(ENCODE.contains("A referent's concept is never null, even when it carries an entity: give the nearest concept id.")); let w = world(); let run = run_over(w.dir.path()); let inv = run.inventory().inventory().unwrap(); -- 2.47.3