crates/weksa/src/context.rs

@@ -59,7 +59,7 @@ Fields:
 - predications: a list of {id, concept, frame, roles, features}; roles is a list of {role, filler}, filler a node id.
 - relations: a list of {relation, from, to, features}.
 - stance: {features}; the speaker's felt affect and restraint, never the affect the line displays.
-Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line.
+Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A referent's concept is never null, even when it carries an entity: give the nearest concept id.
 A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": label} or {\"Flag\": true or false}, as the inventory lists the feature's kind.";
 
 /// The instructions a lowering prompt starts with: the lowerer's whole brief.

crates/weksa/src/encode.rs

@@ -94,24 +94,27 @@ impl DraftClass {
             Category::Syntax => DraftClass::Syntax,
             Category::Eof => DraftClass::Eof,
             Category::Io => DraftClass::Other,
-            // Only the leading words of serde's message are read, and only to
-            // pick one of these classes; the message is not kept.
-            Category::Data => {
-                let msg = e.to_string();
-                if msg.starts_with("missing field") {
-                    DraftClass::MissingField
-                } else if msg.starts_with("unknown field") {
-                    DraftClass::UnknownField
-                } else if msg.starts_with("invalid type") {
-                    DraftClass::InvalidType
-                } else if msg.starts_with("invalid value") {
-                    DraftClass::InvalidValue
-                } else if msg.starts_with("not a ") {
-                    DraftClass::BadForm
-                } else {
-                    DraftClass::Other
-                }
-            }
+            Category::Data => Self::of_data(&e.to_string()),
+        }
+    }
+
+    /// serde_json offers no kind for a data error: its message opens with serde's
+    /// own fixed words, or with a [`crate::form::FormError`]'s `not a `. Only
+    /// those words are read, to pick a class; the message is not kept, and any
+    /// other opening is [`DraftClass::Other`].
+    fn of_data(msg: &str) -> Self {
+        if msg.starts_with("missing field") {
+            DraftClass::MissingField
+        } else if msg.starts_with("unknown field") {
+            DraftClass::UnknownField
+        } else if msg.starts_with("invalid type") {
+            DraftClass::InvalidType
+        } else if msg.starts_with("invalid value") {
+            DraftClass::InvalidValue
+        } else if msg.starts_with("not a ") {
+            DraftClass::BadForm
+        } else {
+            DraftClass::Other
         }
     }
 }
@@ -319,15 +322,26 @@ mod tests {
     }
 
     #[test]
-    fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() {
+    fn draft_class_per_real_serde_error() {
+        // Each class is fed by the error serde_json raises for a real answer.
         let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class;
         assert_eq!(class("not json"), DraftClass::Syntax);
         assert_eq!(class("{\"referents\": ["), DraftClass::Eof);
         assert_eq!(class("{}"), DraftClass::MissingField);
-        // The classes serde's own messages open with, read off a plain error.
-        let of = |r: Result<u8, serde_json::Error>| DraftClass::of(&r.unwrap_err());
-        assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue);
-        assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType);
+        assert_eq!(class("{\"bogus\": 1}"), DraftClass::UnknownField);
+        assert_eq!(class("{\"confidence\": \"x\"}"), DraftClass::InvalidType);
+        assert_eq!(class("{\"confidence\": -1}"), DraftClass::InvalidValue);
+        let bad_id = r#"{"referents": [{"id": "entity:x"}]}"#;
+        assert_eq!(class(bad_id), DraftClass::BadForm);
+        // A data error opening with words this reader does not know is Other.
+        assert_eq!(
+            DraftClass::of_data("something serde never said"),
+            DraftClass::Other
+        );
+        assert_eq!(DraftClass::of_data(""), DraftClass::Other);
+        // None keeps a token of the answer.
+        let secret = r#"{"confidence": "canary-token"}"#;
+        assert!(!format!("{:?}", PacketDraft::parse(secret).unwrap_err()).contains("canary"));
         // Every code is distinct and none can hold answer text: they are fixed.
         let all = [
             DraftClass::Syntax,

crates/weksa/src/run.rs

@@ -587,7 +587,16 @@ impl RunStores {
         reach: LowererReach,
         answerer_family: ModelFamily,
     ) -> Result<Self, RunError> {
-        Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {})
+        Self::create_between(
+            root,
+            label,
+            paths,
+            model,
+            embedder,
+            reach,
+            answerer_family,
+            || {},
+        )
     }
 
     /// `create` with `between` run after the digests are taken and before the
@@ -2125,7 +2134,9 @@ mod tests {
                     entities: &entities,
                 },
                 "fake-model",
-                "bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+                "bag",
+                crate::run::LowererReach::ToolLessRelay,
+                ModelFamily::Unrecorded,
             )
             .unwrap();
             let mut model = fake();

crates/weksa/tests/cli.rs

@@ -1431,6 +1431,45 @@ fn open_dao_run(d: &std::path::Path) {
     assert!(o.status.success(), "{}", out(&o));
 }
 
+/// The family `loop open` pinned at `root`, read from the run record.
+fn pinned_family(d: &std::path::Path, root: &str, flag: &[&str]) -> weksa::corpus::ModelFamily {
+    let mut args = vec![
+        "loop",
+        "open",
+        root,
+        "--label",
+        "fam",
+        "--answerer",
+        "fake-model",
+        "--lowerer-reach",
+        "tool-less-relay",
+    ];
+    args.extend_from_slice(flag);
+    let o = weksa(d, &args);
+    assert!(o.status.success(), "{}", out(&o));
+    weksa::run::RunStores::open(&d.join(root))
+        .unwrap()
+        .record()
+        .answerer_family
+}
+
+#[test]
+fn loop_open_pins_answerer_family() {
+    use weksa::corpus::ModelFamily;
+    let d = tempfile::tempdir().unwrap();
+    dao_stores(d.path());
+    assert_eq!(
+        pinned_family(d.path(), "a", &["--answerer-family", "anthropic"]),
+        ModelFamily::Anthropic
+    );
+    assert_eq!(
+        pinned_family(d.path(), "b", &["--answerer-family", "mistral"]),
+        ModelFamily::Mistral
+    );
+    // Omitted, the family stays unrecorded and a round prints no own-family line.
+    assert_eq!(pinned_family(d.path(), "c", &[]), ModelFamily::Unrecorded);
+}
+
 #[test]
 fn loop_cycle_needs_a_floor() {
     let d = tempfile::tempdir().unwrap();

crates/weksa/tests/common/mod.rs

@@ -131,7 +131,9 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
             entities: &dir.join("entities.cc"),
         },
         "fake-model",
-        "bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded,
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        weksa::corpus::ModelFamily::Unrecorded,
     )
     .unwrap()
 }

crates/weksa/tests/context.rs

@@ -694,6 +694,7 @@ mod lowering {
 #[test]
 fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() {
     assert!(ENCODE.contains("family prefix included: concept:7, never 7"));
+    assert!(ENCODE.contains("A referent's concept is never null, even when it carries an entity: give the nearest concept id."));
     let w = world();
     let run = run_over(w.dir.path());
     let inv = run.inventory().inventory().unwrap();

crates/weksa/tests/golf_loop.rs

@@ -48,7 +48,9 @@ fn a_run_pins_its_stores_and_refuses_drift() {
                 entities: &e,
             },
             model,
-            "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+            "bag",
+            weksa::run::LowererReach::ToolLessRelay,
+            ModelFamily::Unrecorded,
         )
     };
 
@@ -149,7 +151,9 @@ fn a_run_works_on_its_own_copy_of_the_inventory() {
             entities: &e,
         },
         "fake-model",
-        "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
     )
     .unwrap();
 
@@ -224,7 +228,9 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() {
                 entities: &dao.entities,
             },
             "fake-model",
-            "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+            "bag",
+            weksa::run::LowererReach::ToolLessRelay,
+            ModelFamily::Unrecorded,
         )
     };
 
@@ -269,7 +275,9 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() {
                 entities,
             },
             "fake-model",
-            "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+            "bag",
+            weksa::run::LowererReach::ToolLessRelay,
+            ModelFamily::Unrecorded,
         )
     };
     // A directory opens for reading and fails mid-copy, after the destination exists.
@@ -382,14 +390,33 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() {
         entities: &e,
     };
     assert_eq!(
-        RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(),
+        RunStores::create_between(
+            &root,
+            "t",
+            paths(),
+            "fake-model",
+            "bag",
+            weksa::run::LowererReach::ToolLessRelay,
+            ModelFamily::Unrecorded,
+            || more(&mut w)
+        )
+        .err(),
         Some(RunError::Drift(DriftOf::Corpus))
     );
     assert!(!root.join("inventory.cc").exists());
     // Not the copy and not the lock the store's open left beside it.
     assert!(!root.join("inventory.cc.lock").exists());
     assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun));
-    RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap();
+    RunStores::create(
+        &root,
+        "t",
+        paths(),
+        "fake-model",
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
+    )
+    .unwrap();
 }
 
 /// Runs `change` in the window between the pins and the open, at `open` and at
@@ -412,9 +439,16 @@ fn window(
         inventory: &i,
         entities: &e,
     };
-    let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || {
-        change(&mut w, &root)
-    })
+    let created = RunStores::create_between(
+        &root,
+        "t",
+        paths,
+        "fake-model",
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
+        || change(&mut w, &root),
+    )
     .err();
     let left = root.join("inventory.cc").exists() || root.join("inventory.cc.lock").exists();
     (opened, created, left, RunStores::open(&root).err())
@@ -829,7 +863,9 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao {
             entities: &entities,
         },
         "fake-model",
-        "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
     )
     .unwrap();
     Dao {
@@ -1478,17 +1514,41 @@ fn errors_echo_no_input() {
         inventory: &missing,
         entities: &missing,
     };
-    let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
-        .err()
-        .unwrap();
+    let e = RunStores::create(
+        &root,
+        "ok",
+        stores,
+        "m",
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
+    )
+    .err()
+    .unwrap();
     note(&e, &e);
-    let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
-        .err()
-        .unwrap();
+    let e = RunStores::create(
+        &root,
+        &format!("{canary}!"),
+        stores,
+        "m",
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
+    )
+    .err()
+    .unwrap();
     note(&e, &e);
-    let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
-        .err()
-        .unwrap();
+    let e = RunStores::create(
+        &root,
+        "ok",
+        stores,
+        &format!("{canary}\n"),
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
+    )
+    .err()
+    .unwrap();
     note(&e, &e);
     let w = world();
     let run = run_over(w.dir.path());
@@ -1520,7 +1580,9 @@ fn errors_echo_no_input() {
             entities: &dao.entities,
         },
         "fake-model",
-        "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
     )
     .unwrap();
     let mut model = cycle_fake(&format!("{{\"{canary}\":1}}"));
@@ -2450,7 +2512,9 @@ fn a_changed_head_is_drift() {
             entities: &dao.entities,
         },
         "fake-model",
-        "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
+        "bag",
+        weksa::run::LowererReach::ToolLessRelay,
+        ModelFamily::Unrecorded,
     )
     .unwrap();
     assert_eq!(
@@ -2769,8 +2833,16 @@ fn a_summary_line_says_how_lowerings_were_answered() {
         inventory: &dao.inventory,
         entities: &dao.entities,
     };
-    RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded)
-        .unwrap();
+    RunStores::create(
+        &root,
+        "b",
+        paths,
+        "fake-model",
+        "bag",
+        LowererReach::BriefedSubagent,
+        ModelFamily::Unrecorded,
+    )
+    .unwrap();
     let run = RunStores::open(&root).unwrap();
     assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent);
     assert_eq!(
@@ -3002,7 +3074,11 @@ fn parse_reasons_by_class() {
     for (code, answer) in cases {
         let (dao, r) = round_with_item_1(answer);
         assert_eq!(reasons(&r), [(code, 1)], "{code}");
-        assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}");
+        assert_eq!(
+            r.train()[0].outcome,
+            Outcome::Refused(Stage::Parse),
+            "{code}"
+        );
         let line = dao.run.round_line(&r);
         assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}");
     }
@@ -3365,3 +3441,122 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdra
     let unnamed = m.seen.iter().filter(|t| t.contains("work: not published")).count();
     assert_eq!((named, unnamed), (2, 1));
 }
+
+/// Answers every prompt with text that is not a draft, so a round measures only
+/// what it counts before admission.
+struct NotADraft;
+
+impl weksa::model::ModelPort for NotADraft {
+    fn model(&self) -> &str {
+        "not-a-draft"
+    }
+    fn answer(&mut self, _: &weksa::context::Prompt) -> Result<String, weksa::model::PortError> {
+        Ok("not a draft".to_string())
+    }
+}
+
+struct OneBag;
+
+impl weksa::score::Embedder for OneBag {
+    fn model(&self) -> &str {
+        "bag"
+    }
+    fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
+        Ok(texts.iter().map(|_| vec![1.0]).collect())
+    }
+}
+
+/// A corpus of every shape a family count must tell apart, and the own-family
+/// count of a round over it by an answerer of `family`.
+fn mixed_family_round(family: ModelFamily) -> u32 {
+    use common::{cast, group_in_bucket, item_in};
+    use weksa::corpus::Source;
+    use weksa::run::{LowererReach, RoundOutcome, round};
+
+    let mut w = world();
+    let written = |family| Source::ModelWritten {
+        family,
+        generator: None,
+        prompt_digest: None,
+    };
+    let public_domain = Source::PublicDomain {
+        work: "w".into(),
+        author: "a".into(),
+        edition_url: "https://example.org/e".into(),
+        licence: "PD".into(),
+        jurisdiction: "US".into(),
+        translator: "t".into(),
+        edition_year: 1900,
+    };
+    let (train, validation, test) = (
+        group_in_bucket(10),
+        group_in_bucket(50),
+        group_in_bucket(90),
+    );
+    let items = [
+        // Offered, Anthropic: two of them count for Anthropic.
+        ("item:mix:1", &train, written(ModelFamily::Anthropic)),
+        ("item:mix:2", &validation, written(ModelFamily::Anthropic)),
+        // Offered, other families or no model at all: never Anthropic's.
+        ("item:mix:3", &train, written(ModelFamily::OpenAi)),
+        ("item:mix:4", &train, written(ModelFamily::Unrecorded)),
+        ("item:mix:5", &train, public_domain),
+        ("item:mix:6", &train, cast()),
+        // Model-written in a Test group: in no split, never offered.
+        ("item:mix:7", &test, written(ModelFamily::Anthropic)),
+        // Withdrawn after admission: never offered.
+        ("item:mix:8", &train, written(ModelFamily::Anthropic)),
+    ];
+    let entities = common::entities_in(w.dir.path());
+    for (key, group, source) in items {
+        let text = format!("Line {key}, distinct.");
+        w.corpus
+            .add(item_in(key, &text, group, source), &entities)
+            .unwrap();
+    }
+    w.corpus.withdraw("item:mix:8").unwrap();
+    let dir = w.dir.path().to_path_buf();
+    for name in ["c.cc", "i.cc", "entities.cc"] {
+        assert!(dir.join(name).exists(), "{name}");
+    }
+    let mut run = RunStores::create(
+        &dir.join("run"),
+        "mix",
+        RunPaths {
+            corpus: &dir.join("c.cc"),
+            inventory: &dir.join("i.cc"),
+            entities: &dir.join("entities.cc"),
+        },
+        "not-a-draft",
+        "bag",
+        LowererReach::ToolLessRelay,
+        family,
+    )
+    .unwrap();
+    let scorer = weksa::score::Scorer::new(OneBag);
+    match round(&mut run, &mut NotADraft, &scorer).unwrap() {
+        RoundOutcome::Recorded(r) => {
+            // Seven items were offered: the world's own cast item and mix 1 to 6;
+            // the Test-group and withdrawn items were not.
+            assert_eq!(
+                r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
+                7,
+                "offered items"
+            );
+            r.own_family()
+        }
+        RoundOutcome::Unanswered { .. } => panic!("answered"),
+    }
+}
+
+#[test]
+fn own_family_counts_offered_items_of_the_answerer_family() {
+    // Two live Train/Validation Anthropic items; the Test-group and withdrawn
+    // Anthropic items are not offered, so they are not counted.
+    assert_eq!(mixed_family_round(ModelFamily::Anthropic), 2);
+    assert_eq!(mixed_family_round(ModelFamily::OpenAi), 1);
+    // Google wrote nothing here; an unrecorded answerer counts no item, though
+    // an offered item is itself unrecorded.
+    assert_eq!(mixed_family_round(ModelFamily::Google), 0);
+    assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0);
+}

crates/weksa/tests/original_corpus.rs

@@ -99,7 +99,8 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
         },
         "m",
         "bag",
-        weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded
+        weksa::run::LowererReach::BriefedSubagent,
+        ModelFamily::Unrecorded,
     )
     .unwrap();
     let mut model = Unrelated(vec![]);
@@ -121,10 +122,7 @@ impl weksa::model::ModelPort for Nope {
     fn model(&self) -> &str {
         "nope"
     }
-    fn answer(
-        &mut self,
-        _: &weksa::context::Prompt,
-    ) -> Result<String, weksa::model::PortError> {
+    fn answer(&mut self, _: &weksa::context::Prompt) -> Result<String, weksa::model::PortError> {
         Ok("not a draft".to_string())
     }
 }