GameCult/Weksa
Watch
9
0
Fork
You've already forked Weksa
 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

Round records and lines carry parse refusal reasons and an own-family label #41

Merged
weksa merged 2 commits from weksa-lowering/parse-reason into main 2026-11-10 17:28:09 +00:00
Conversation 0 Commits 2 Files changed 12 +604 -39
12 changed files with 604 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/weksa/src/context.rs
View file

@ -86,7 +86,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
/// let t = run.train().unwrap();
/// assert!(t.is_empty());
/// ```
@ -100,7 +100,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
/// let t = run.train().unwrap();
/// let _u = t.clone();
/// ```
@ -111,7 +111,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
/// let t = run.train().unwrap();
/// s(&t);
/// ```
@ -317,7 +317,7 @@ impl EditFeedback {
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
/// let t = run.train().unwrap();
/// let inv = run.inventory().inventory().unwrap();
/// let feedback = weksa::context::EditFeedback::none();
@ -335,7 +335,7 @@ impl EditFeedback {
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
/// let t = run.train().unwrap();
/// let inv = run.inventory().inventory().unwrap();
/// let feedback = weksa::context::EditFeedback::none();
@ -348,7 +348,7 @@ impl EditFeedback {
/// # let p = |n: &str| dir.path().join(n);
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
/// let t = run.train().unwrap();
/// let inv = run.inventory().inventory().unwrap();
/// let feedback = weksa::context::EditFeedback::none();

8
crates/weksa/src/corpus.rs
View file

@ -43,7 +43,10 @@ pub enum Source {
translator: String,
edition_year: u16,
},
/// Labelled model output; never in the test split, never judged by its own family.
/// Labelled model output; never in the test split. The rule that it is
/// never judged by its own family binds model judges and the split; a
/// deterministic scorer refuses nothing, so the loop labels (the round
/// counts items of the answerer's family) and does not forbid.
/// `family` is `Unrecorded` when nothing records one. `generator` is the
/// string the fixture or run wrote, kept verbatim.
ModelWritten {
@ -54,7 +57,7 @@ pub enum Source {
}
/// The family of the model that wrote a line.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum ModelFamily {
Anthropic,
OpenAi,
@ -63,6 +66,7 @@ pub enum ModelFamily {
Meta,
Mistral,
/// Nothing records which family wrote the line.
#[default]
Unrecorded,
}

93
crates/weksa/src/encode.rs
View file

@ -47,6 +47,73 @@ pub struct PacketDraft {
pub struct DraftError {
pub line: usize,
pub column: usize,
/// What kind of reading failure, from serde's own fixed vocabulary. It
/// never holds a token of the answer.
pub class: DraftClass,
}
/// Why an answer is not a draft: a closed set of classes, never the offending
/// text, so a record may keep it for a held-out item.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum DraftClass {
/// Not JSON.
Syntax,
/// JSON that ends early.
Eof,
MissingField,
UnknownField,
/// A value of the wrong JSON type, such as a bare integer for an id.
InvalidType,
/// A value of the right type that is not one of the field's, such as a
/// number out of range.
InvalidValue,
/// A string that is not the form its field names: a node id spelled
/// `entity:x`, an id of the wrong kind ([`crate::form::FormError`]).
BadForm,
Other,
}
impl DraftClass {
/// The code a round record keeps.
pub fn code(self) -> &'static str {
match self {
DraftClass::Syntax => "syntax",
DraftClass::Eof => "eof",
DraftClass::MissingField => "missing-field",
DraftClass::UnknownField => "unknown-field",
DraftClass::InvalidType => "invalid-type",
DraftClass::InvalidValue => "invalid-value",
DraftClass::BadForm => "bad-form",
DraftClass::Other => "other",
}
}
fn of(e: &serde_json::Error) -> Self {
use serde_json::error::Category;
match e.classify() {
Category::Syntax => DraftClass::Syntax,
Category::Eof => DraftClass::Eof,
Category::Io => DraftClass::Other,
// Only the leading words of serde's message are read, and only to
// pick one of these classes; the message is not kept.
Category::Data => {
let msg = e.to_string();
if msg.starts_with("missing field") {
DraftClass::MissingField
} else if msg.starts_with("unknown field") {
DraftClass::UnknownField
} else if msg.starts_with("invalid type") {
DraftClass::InvalidType
} else if msg.starts_with("invalid value") {
DraftClass::InvalidValue
} else if msg.starts_with("not a ") {
DraftClass::BadForm
} else {
DraftClass::Other
}
}
}
}
}
impl fmt::Display for DraftError {
@ -72,6 +139,7 @@ fn from_json<T: DeserializeOwned>(answer: &str) -> Result<T, DraftError> {
serde_json::from_str(answer.trim()).map_err(|e| DraftError {
line: e.line(),
column: e.column(),
class: DraftClass::of(&e),
})
}
@ -250,6 +318,31 @@ mod tests {
CorpusItemId::parse("item:daodejing:1").unwrap()
}
#[test]
fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() {
let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class;
assert_eq!(class("not json"), DraftClass::Syntax);
assert_eq!(class("{\"referents\": ["), DraftClass::Eof);
assert_eq!(class("{}"), DraftClass::MissingField);
// The classes serde's own messages open with, read off a plain error.
let of = |r: Result<u8, serde_json::Error>| DraftClass::of(&r.unwrap_err());
assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue);
assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType);
// Every code is distinct and none can hold answer text: they are fixed.
let all = [
DraftClass::Syntax,
DraftClass::Eof,
DraftClass::MissingField,
DraftClass::UnknownField,
DraftClass::InvalidType,
DraftClass::InvalidValue,
DraftClass::BadForm,
DraftClass::Other,
];
let codes: std::collections::BTreeSet<_> = all.iter().map(|c| c.code()).collect();
assert_eq!(codes.len(), all.len());
}
#[test]
fn ids_are_pinned_digests_of_their_inputs() {
let encoder = encoder_id("fake-model", "abcd");

6
crates/weksa/src/main.rs
View file

@ -92,6 +92,10 @@ enum LoopCommand {
/// rounds only) or tool-less-relay. Pinned in the run's record.
#[arg(long)]
lowerer_reach: LowererReach,
/// The family of the answering model; rounds count the corpus items
/// that family wrote. Defaults to unrecorded.
#[arg(long, default_value = "unrecorded")]
answerer_family: ModelFamilyArg,
},
/// Run one round of the run at ROOT. The stores are read only from the
/// run's record: the global --corpus, --inventory and --entities are not
@ -744,6 +748,7 @@ fn main() -> Result<()> {
answerer,
embed_model,
lowerer_reach,
answerer_family,
}) => {
let run = RunStores::create(
&root,
@ -756,6 +761,7 @@ fn main() -> Result<()> {
&answerer,
&embed_model,
lowerer_reach,
answerer_family.into(),
)?;
println!("opened run:{}", run.record().label);
}

90
crates/weksa/src/run.rs
View file

@ -35,7 +35,7 @@ use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems};
use crate::corpus::{CorpusItem, CorpusItemId, Source};
use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source};
use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id};
use crate::form::{is_attestation, is_slug};
use crate::model::{ModelPort, PortError};
@ -107,6 +107,12 @@ pub struct RunRecord {
/// How the run's lowering prompts are answered, set once at open.
#[cultcache(key = 12)]
pub lowerer_reach: LowererReach,
/// The family of the model answering the run's encode and lowering
/// prompts, set once at open; `Unrecorded` when the opener did not say
/// and for a run opened before this was kept.
#[cultcache(key = 13, default)]
#[schemars(default)]
pub answerer_family: ModelFamily,
}
/// How a run's lowering prompts reach the model that answers them (rulings
@ -224,6 +230,8 @@ pub struct RefusalCounts {
/// inventory_digest: String::new(),
/// cycle: None,
/// admit_rules: vec![],
/// parse_reasons: vec![],
/// own_family: 0,
/// }
/// }
/// ```
@ -276,9 +284,22 @@ pub struct RoundRecord {
cycle: Option<u32>,
/// The rules admission refused items under, in rule order; every Train
/// and Validation item refused at `Stage::Admit` is counted once per rule.
#[cultcache(key = 18)]
#[cultcache(key = 18, default)]
#[schemars(default)]
admit_rules: Vec<RuleCount>,
/// Why items were refused at `Stage::Parse`, in code order, over Train and
/// Validation: a [`DraftClass`] code (`rule`), or `unreadable` for an
/// item with no answer that read as text. Never a token of an answer.
#[cultcache(key = 19, default)]
#[schemars(default)]
parse_reasons: Vec<RuleCount>,
/// Offered (live Train and Validation) `ModelWritten` items whose writer
/// family is the run's answerer family. The corpus rule binds
/// model judges; the scorer is deterministic, so nothing is refused for
/// this: the round labels it.
#[cultcache(key = 20, default)]
#[schemars(default)]
own_family: u32,
}
impl RoundRecord {
@ -336,6 +357,12 @@ impl RoundRecord {
pub fn admit_rules(&self) -> &[RuleCount] {
&self.admit_rules
}
pub fn parse_reasons(&self) -> &[RuleCount] {
&self.parse_reasons
}
pub fn own_family(&self) -> u32 {
self.own_family
}
/// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split,
/// `scored` those that lowered; a mean counts a refused item as 0.
@ -369,16 +396,24 @@ impl RoundRecord {
x.admit,
x.lower,
);
if self.admit_rules.is_empty() {
line
} else {
let rules: Vec<String> = self
.admit_rules
let codes = |counts: &[RuleCount]| {
counts
.iter()
.map(|r| format!("{}x{}", r.rule, r.count))
.collect();
format!("{line} admit-rules={}", rules.join(","))
.collect::<Vec<_>>()
.join(",")
};
let mut line = line;
if !self.parse_reasons.is_empty() {
line = format!("{line} parse-reasons={}", codes(&self.parse_reasons));
}
if !self.admit_rules.is_empty() {
line = format!("{line} admit-rules={}", codes(&self.admit_rules));
}
if self.own_family > 0 {
line = format!("{line} own-family {}", self.own_family);
}
line
}
}
@ -550,13 +585,15 @@ impl RunStores {
model: &str,
embedder: &str,
reach: LowererReach,
answerer_family: ModelFamily,
) -> Result<Self, RunError> {
Self::create_between(root, label, paths, model, embedder, reach, || {})
Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {})
}
/// `create` with `between` run after the digests are taken and before the
/// stores are opened: the seam that lets a test change a store in that window.
#[doc(hidden)]
#[allow(clippy::too_many_arguments)] // `create`'s six plus the test seam
pub fn create_between(
root: &Path,
label: &str,
@ -564,6 +601,7 @@ impl RunStores {
model: &str,
embedder: &str,
reach: LowererReach,
answerer_family: ModelFamily,
between: impl FnOnce(),
) -> Result<Self, RunError> {
if !is_slug(label) {
@ -592,6 +630,7 @@ impl RunStores {
chrf_version: CHRF_VERSION.to_string(),
inventory_digest: String::new(),
lowerer_reach: reach,
answerer_family,
};
// The run works on its own copy; the caller's store is never opened again.
let head = root.join(INVENTORY_FILE);
@ -1024,13 +1063,21 @@ fn score_round<M: ModelPort, E: Embedder>(
let mut items: Vec<CorpusItem> = Vec::new();
let mut fates: Vec<Fate> = Vec::new();
let mut admit_rules: BTreeMap<String, u32> = BTreeMap::new();
let mut parse_reasons: BTreeMap<String, u32> = BTreeMap::new();
// Phase 1 ends at admission: each admitted view waits for its lowering.
let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new();
for (key, answer) in &answers {
let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?;
let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?;
let fate = match answer.as_deref().map(PacketDraft::parse) {
None | Some(Err(_)) => Fate::Refused(Stage::Parse, None),
None => {
*parse_reasons.entry("unreadable".to_string()).or_default() += 1;
Fate::Refused(Stage::Parse, None)
}
Some(Err(e)) => {
*parse_reasons.entry(e.class.code().to_string()).or_default() += 1;
Fate::Refused(Stage::Parse, None)
}
Some(Ok(draft)) => {
let packet = draft.into_packet(&id, rev, &encoder);
let packet_key = packet.id.to_string();
@ -1172,6 +1219,15 @@ fn score_round<M: ModelPort, E: Embedder>(
}
}
let own_family = match run.record.answerer_family {
ModelFamily::Unrecorded => 0,
family => items
.iter()
.filter(|item| {
matches!(&item.source, Source::ModelWritten { family: f, .. } if *f == family)
})
.count() as u32,
};
let n = run.rounds().map_err(store)?.len() as u32 + 1;
let record = RoundRecord {
n,
@ -1195,6 +1251,11 @@ fn score_round<M: ModelPort, E: Embedder>(
.into_iter()
.map(|(rule, count)| RuleCount { rule, count })
.collect(),
parse_reasons: parse_reasons
.into_iter()
.map(|(rule, count)| RuleCount { rule, count })
.collect(),
own_family,
};
Ok(RoundOutcome::Recorded(record))
}
@ -1628,6 +1689,8 @@ mod tests {
inventory_digest: String::new(),
cycle: None,
admit_rules: vec![],
parse_reasons: vec![],
own_family: 0,
};
assert_eq!(r.n(), 1);
}
@ -1844,6 +1907,8 @@ mod tests {
inventory_digest: digest.to_string(),
cycle: None,
admit_rules: vec![],
parse_reasons: vec![],
own_family: 0,
}
}
@ -1861,6 +1926,7 @@ mod tests {
chrf_version: String::new(),
inventory_digest: digest.to_string(),
lowerer_reach: LowererReach::ToolLessRelay,
answerer_family: ModelFamily::Unrecorded,
}
}
@ -2059,7 +2125,7 @@ mod tests {
entities: &entities,
},
"fake-model",
"bag", crate::run::LowererReach::ToolLessRelay,
"bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
.unwrap();
let mut model = fake();

4
crates/weksa/tests/cli.rs
View file

@ -1346,6 +1346,8 @@ fn loop_open_and_unanswered_round() {
"fake-model",
"--lowerer-reach",
"tool-less-relay",
"--answerer-family",
"open-ai",
],
)
};
@ -1395,7 +1397,7 @@ fn loop_open_and_unanswered_round() {
assert!(second.status.success(), "{}", out(&second));
assert_eq!(
out(&second),
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 lowering=tool-less-relay\n"
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n"
);
// The global store flags are not read by loop round: a drifted store is refused.

2
crates/weksa/tests/common/mod.rs
View file

@ -131,7 +131,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
entities: &dir.join("entities.cc"),
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded,
)
.unwrap()
}

305
crates/weksa/tests/golf_loop.rs
View file

@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
use common::{ent, run_over, world};
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
use weksa::corpus::ModelFamily;
use weksa::run::{DriftOf, RunError, RunPaths, RunRecord, RunStores};
use weksa::split::fnv1a64;
@ -47,7 +48,7 @@ fn a_run_pins_its_stores_and_refuses_drift() {
entities: &e,
},
model,
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
};
@ -148,7 +149,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() {
entities: &e,
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
.unwrap();
@ -223,7 +224,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() {
entities: &dao.entities,
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
};
@ -268,7 +269,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() {
entities,
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
};
// A directory opens for reading and fails mid-copy, after the destination exists.
@ -381,14 +382,14 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() {
entities: &e,
};
assert_eq!(
RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || more(&mut w)).err(),
RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(),
Some(RunError::Drift(DriftOf::Corpus))
);
assert!(!root.join("inventory.cc").exists());
// Not the copy and not the lock the store's open left beside it.
assert!(!root.join("inventory.cc.lock").exists());
assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun));
RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay).unwrap();
RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap();
}
/// Runs `change` in the window between the pins and the open, at `open` and at
@ -411,7 +412,7 @@ fn window(
inventory: &i,
entities: &e,
};
let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || {
let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || {
change(&mut w, &root)
})
.err();
@ -828,7 +829,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao {
entities: &entities,
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
.unwrap();
Dao {
@ -1477,15 +1478,15 @@ fn errors_echo_no_input() {
inventory: &missing,
entities: &missing,
};
let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay)
let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
.err()
.unwrap();
note(&e, &e);
let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay)
let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
.err()
.unwrap();
note(&e, &e);
let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay)
let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
.err()
.unwrap();
note(&e, &e);
@ -1519,7 +1520,7 @@ fn errors_echo_no_input() {
entities: &dao.entities,
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
.unwrap();
let mut model = cycle_fake(&format!("{{\"{canary}\":1}}"));
@ -2449,7 +2450,7 @@ fn a_changed_head_is_drift() {
entities: &dao.entities,
},
"fake-model",
"bag", weksa::run::LowererReach::ToolLessRelay,
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
)
.unwrap();
assert_eq!(
@ -2768,7 +2769,7 @@ fn a_summary_line_says_how_lowerings_were_answered() {
inventory: &dao.inventory,
entities: &dao.entities,
};
RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent)
RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded)
.unwrap();
let run = RunStores::open(&root).unwrap();
assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent);
@ -2947,6 +2948,282 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() {
assert!(!dao.run.round_line(&r).contains("admit-rules"));
}
/// Item 1's good draft as JSON, changed by `change`, as the model's answer.
fn mangled(change: impl FnOnce(&mut serde_json::Value)) -> String {
let mut v: serde_json::Value = serde_json::from_str(&json(&draft_of(1))).unwrap();
change(&mut v);
v.to_string()
}
/// The round record of a Dao run whose item 1 is answered with `answer` and
/// the rest well.
fn round_with_item_1(answer: String) -> (Dao, RoundRecord) {
let mut dao = dao();
let mut answers = good_answers();
answers[0].1 = answer;
let mut model = fake(&answers);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
(dao, r)
}
fn reasons(r: &RoundRecord) -> Vec<(&str, u32)> {
r.parse_reasons()
.iter()
.map(|c| (c.rule.as_str(), c.count))
.collect()
}
#[test]
fn parse_reasons_by_class() {
let good = json(&draft_of(1));
let truncated = good[..good.len() / 2].to_string();
let cases: Vec<(&str, String)> = vec![
("syntax", "not json at all".to_string()),
("eof", truncated),
(
"invalid-type",
mangled(|v| v["referents"][0]["concept"] = serde_json::json!(7)),
),
(
"bad-form",
mangled(|v| v["referents"][0]["concept"] = serde_json::json!("entity:x")),
),
(
"unknown-field",
mangled(|v| v["surprise"] = serde_json::json!(1)),
),
(
"missing-field",
mangled(|v| {
v.as_object_mut().unwrap().remove("referents");
}),
),
];
for (code, answer) in cases {
let (dao, r) = round_with_item_1(answer);
assert_eq!(reasons(&r), [(code, 1)], "{code}");
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}");
let line = dao.run.round_line(&r);
assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}");
}
// A round nothing refused at Parse carries and prints none.
let (dao, r) = round_with_item_1(good);
assert!(r.parse_reasons().is_empty());
assert!(!dao.run.round_line(&r).contains("parse-reasons"));
}
#[test]
fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() {
let mut dao = dao();
let expected = prompts(&dao.run);
let answers = keyed(&good_answers());
let round1 = dao.root.join("round-1");
std::fs::create_dir_all(&round1).unwrap();
for (key, p) in &expected {
let bytes = match key.as_str() {
// Train: not UTF-8, so no answer parsed from the file.
"item:daodejing:1" => vec![0xff, 0xfe],
// Validation: not JSON.
"item:daodejing:2" => b"nope".to_vec(),
_ => answers[key].clone().into_bytes(),
};
std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap();
}
let table = table_english();
for (key, p) in lowering_prompts(&dao, dao.run.inventory())
.iter()
.filter(|(key, _)| key == "item:daodejing:3")
{
let answer = serde_json::json!({ "text": table[key] }).to_string();
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1, "fake-model".into());
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]);
}
#[test]
fn parse_reasons_never_echo_answer() {
let canary = "CANARY-7f3a-held-out";
// The canary sits where serde would quote it: an unknown field name, an
// invalid value, and the raw text of a non-JSON answer.
let answers = [
mangled(|v| v[canary] = serde_json::json!(1)),
mangled(|v| v["referents"][0]["concept"] = serde_json::json!(canary)),
format!("{canary} is not json"),
];
for answer in answers {
// Item 2 is Validation: the held-out item carries the canary.
let mut dao = dao();
let mut a = good_answers();
a[1].1 = answer;
let mut model = fake(&a);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(r.validation_refused().parse, 1);
let line = dao.run.round_line(&r);
assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}");
let bytes = std::fs::read(dao.root.join("run.cc")).unwrap();
assert!(!bytes.windows(6).any(|w| w == b"CANARY"));
assert!(!format!("{r:?}").contains("CANARY"));
}
}
/// A run record as written before `answerer_family` (key 13) existed.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_run")]
struct PreFamilyRun {
#[cultcache(key = 0)]
label: String,
#[cultcache(key = 1)]
corpus: String,
#[cultcache(key = 2)]
inventory: String,
#[cultcache(key = 3)]
entities: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
prompt_digest: String,
#[cultcache(key = 8)]
model: String,
#[cultcache(key = 9)]
embedder: String,
#[cultcache(key = 10)]
chrf_version: String,
#[cultcache(key = 11)]
inventory_digest: String,
#[cultcache(key = 12)]
lowerer_reach: weksa::run::LowererReach,
}
/// A round record as written before `parse_reasons` and `own_family`, and
/// before `admit_rules` had a default.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_round")]
struct LegacyRound {
#[cultcache(key = 0)]
n: u32,
#[cultcache(key = 1)]
inventory_rev: u32,
#[cultcache(key = 2)]
prompt_digest: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
model: String,
#[cultcache(key = 7)]
embedder: String,
#[cultcache(key = 8)]
chrf_version: String,
#[cultcache(key = 9)]
calls: Vec<weksa::run::Call>,
#[cultcache(key = 10)]
train: Vec<weksa::run::TrainRow>,
#[cultcache(key = 11)]
train_mean: weksa::score::SplitMean,
#[cultcache(key = 12)]
validation_mean: weksa::score::SplitMean,
#[cultcache(key = 13)]
validation_refused: weksa::run::RefusalCounts,
#[cultcache(key = 14)]
train_scored: u32,
#[cultcache(key = 15)]
validation_scored: u32,
#[cultcache(key = 16)]
inventory_digest: String,
#[cultcache(key = 17)]
cycle: Option<u32>,
}
#[test]
fn records_from_before_the_new_fields_decode_with_defaults() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let rec = dao.run.record().clone();
let root = dao.root.clone();
let legacy = LegacyRound {
n: r.n(),
inventory_rev: r.inventory_rev(),
prompt_digest: r.prompt_digest().into(),
corpus_digest: r.corpus_digest().into(),
entity_digest: r.entity_digest().into(),
model: r.model().into(),
embedder: r.embedder().into(),
chrf_version: r.chrf_version().into(),
calls: r.calls().to_vec(),
train: r.train().to_vec(),
train_mean: r.train_mean().clone(),
validation_mean: r.validation_mean().clone(),
validation_refused: r.validation_refused(),
train_scored: r.train_scored(),
validation_scored: r.validation_scored(),
inventory_digest: r.inventory_digest().into(),
cycle: r.cycle(),
};
let run = PreFamilyRun {
label: rec.label.clone(),
corpus: rec.corpus.clone(),
inventory: rec.inventory.clone(),
entities: rec.entities.clone(),
corpus_digest: rec.corpus_digest.clone(),
entity_digest: rec.entity_digest.clone(),
prompt_digest: rec.prompt_digest.clone(),
model: rec.model.clone(),
embedder: rec.embedder.clone(),
chrf_version: rec.chrf_version.clone(),
inventory_digest: rec.inventory_digest.clone(),
lowerer_reach: rec.lowerer_reach,
};
drop(dao.run);
let mut cache = CultCache::new();
cache.register_entry_type::<PreFamilyRun>().unwrap();
cache.register_entry_type::<LegacyRound>().unwrap();
cache
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
.unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &run).unwrap();
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
drop(cache);
let reopened = RunStores::open(&root).unwrap();
assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded);
let rounds = reopened.rounds().unwrap();
assert_eq!(rounds.len(), 1);
assert_eq!(rounds[0].n(), 1);
assert!(rounds[0].admit_rules().is_empty());
assert!(rounds[0].parse_reasons().is_empty());
assert_eq!(rounds[0].own_family(), 0);
}
#[test]
fn the_published_run_and_round_schemas_require_none_of_the_new_fields() {
let published = weksa::schema::published();
let schema = |needle: &str| -> serde_json::Value {
let (_, json) = published
.iter()
.find(|(name, _)| name.contains(needle))
.unwrap();
serde_json::from_str(json).unwrap()
};
for (needle, fields) in [
("loop_round", &["parse_reasons", "own_family"][..]),
("loop_run", &["answerer_family"][..]),
] {
let s = schema(needle);
let required = s["required"].as_array().unwrap();
for f in fields {
assert!(s["properties"][f].is_object(), "{needle} {f}");
assert!(!required.iter().any(|r| r == f), "{needle} {f} is required");
}
}
}
/// Answers a recall probe from the corpus: the item's own second half, a
/// fixed wrong text, or junk.
struct Reciter {

78
crates/weksa/tests/original_corpus.rs
View file

@ -99,7 +99,7 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
},
"m",
"bag",
weksa::run::LowererReach::BriefedSubagent,
weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded
)
.unwrap();
let mut model = Unrelated(vec![]);
@ -112,3 +112,79 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
assert!(model.0.iter().all(|t| t.contains("work: not published")));
assert!(model.0.iter().all(|t| !t.contains("translator:")));
}
/// Answers every prompt with text that is not a draft: nothing is admitted, so
/// the round measures only what is counted before admission.
struct Nope;
impl weksa::model::ModelPort for Nope {
fn model(&self) -> &str {
"nope"
}
fn answer(
&mut self,
_: &weksa::context::Prompt,
) -> Result<String, weksa::model::PortError> {
Ok("not a draft".to_string())
}
}
struct Bag;
impl weksa::score::Embedder for Bag {
fn model(&self) -> &str {
"bag"
}
fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
Ok(texts.iter().map(|_| vec![1.0]).collect())
}
}
fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::RoundRecord) {
use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round};
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original");
let tmp = tempfile::tempdir().unwrap().keep();
for f in ["corpus.cc", "entity.cc", "inventory.cc"] {
std::fs::copy(src.join(f), tmp.join(f)).unwrap();
}
let mut run = RunStores::create(
&tmp.join("run"),
"orig",
RunPaths {
corpus: &tmp.join("corpus.cc"),
inventory: &tmp.join("inventory.cc"),
entities: &tmp.join("entity.cc"),
},
"nope",
"bag",
LowererReach::ToolLessRelay,
family,
)
.unwrap();
let scorer = weksa::score::Scorer::new(Bag);
match round(&mut run, &mut Nope, &scorer).unwrap() {
RoundOutcome::Recorded(r) => (run, r),
RoundOutcome::Unanswered { .. } => panic!("answered"),
}
}
#[test]
fn own_family_counts_and_labels() {
// The original corpus is six Anthropic-written items, all offered.
let (run, r) = original_round(ModelFamily::Anthropic);
assert_eq!(r.own_family(), 6);
assert_eq!(
r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
6,
"every item was refused at Parse and says why"
);
let line = run.round_line(&r);
assert!(line.contains(" own-family 6"), "{line}");
// A different answerer family, or none recorded, labels nothing.
for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] {
let (run, r) = original_round(family);
assert_eq!(r.own_family(), 0);
assert!(!run.round_line(&r).contains("own-family"));
}
}

2
schemas/cultnet/weksa.corpus_item.schema.json
View file

@ -218,7 +218,7 @@
]
},
{
"description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.",
"description": "Labelled model output; never in the test split. The rule that it is\nnever judged by its own family binds model judges and the split; a\ndeterministic scorer refuses nothing, so the loop labels (the round\ncounts items of the answerer's family) and does not forbid.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.",
"type": "object",
"properties": {
"ModelWritten": {

17
schemas/cultnet/weksa.loop_round.schema.json
View file

@ -1,7 +1,7 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "RoundRecord",
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```",
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n }\n}\n```",
"type": "object",
"properties": {
"admit_rules": {
@ -57,6 +57,21 @@
"format": "uint32",
"minimum": 0
},
"own_family": {
"description": "Offered (live Train and Validation) `ModelWritten` items whose writer\nfamily is the run's answerer family. The corpus rule binds\nmodel judges; the scorer is deterministic, so nothing is refused for\nthis: the round labels it.",
"type": "integer",
"format": "uint32",
"default": 0,
"minimum": 0
},
"parse_reasons": {
"description": "Why items were refused at `Stage::Parse`, in code order, over Train and\nValidation: a [`DraftClass`] code (`rule`), or `unreadable` for an\nitem with no answer that read as text. Never a token of an answer.",
"type": "array",
"default": [],
"items": {
"$ref": "#/$defs/RuleCount"
}
},
"prompt_digest": {
"type": "string"
},

26
schemas/cultnet/weksa.loop_run.schema.json
View file

@ -4,6 +4,11 @@
"description": "What a run pinned at open. Key `run:<label>`.",
"type": "object",
"properties": {
"answerer_family": {
"description": "The family of the model answering the run's encode and lowering\nprompts, set once at open; `Unrecorded` when the opener did not say\nand for a run opened before this was kept.",
"$ref": "#/$defs/ModelFamily",
"default": "Unrecorded"
},
"chrf_version": {
"type": "string"
},
@ -77,6 +82,27 @@
"const": "ToolLessRelay"
}
]
},
"ModelFamily": {
"description": "The family of the model that wrote a line.",
"oneOf": [
{
"type": "string",
"enum": [
"Anthropic",
"OpenAi",
"Google",
"Alibaba",
"Meta",
"Mistral"
]
},
{
"description": "Nothing records which family wrote the line.",
"type": "string",
"const": "Unrecorded"
}
]
}
}
}