Merge pull request 'Round records and lines carry parse refusal reasons and an own-family label' (#41) from weksa-lowering/parse-reason into main
This commit is contained in:
commit
0d5dfe53cf
12 changed files with 604 additions and 39 deletions
|
|
@ -86,7 +86,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// assert!(t.is_empty());
|
||||
/// ```
|
||||
|
|
@ -100,7 +100,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let _u = t.clone();
|
||||
/// ```
|
||||
|
|
@ -111,7 +111,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// s(&t);
|
||||
/// ```
|
||||
|
|
@ -317,7 +317,7 @@ impl EditFeedback {
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
|
|
@ -335,7 +335,7 @@ impl EditFeedback {
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
|
|
@ -348,7 +348,7 @@ impl EditFeedback {
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
|
|
|
|||
|
|
@ -43,7 +43,10 @@ pub enum Source {
|
|||
translator: String,
|
||||
edition_year: u16,
|
||||
},
|
||||
/// Labelled model output; never in the test split, never judged by its own family.
|
||||
/// Labelled model output; never in the test split. The rule that it is
|
||||
/// never judged by its own family binds model judges and the split; a
|
||||
/// deterministic scorer refuses nothing, so the loop labels (the round
|
||||
/// counts items of the answerer's family) and does not forbid.
|
||||
/// `family` is `Unrecorded` when nothing records one. `generator` is the
|
||||
/// string the fixture or run wrote, kept verbatim.
|
||||
ModelWritten {
|
||||
|
|
@ -54,7 +57,7 @@ pub enum Source {
|
|||
}
|
||||
|
||||
/// The family of the model that wrote a line.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub enum ModelFamily {
|
||||
Anthropic,
|
||||
OpenAi,
|
||||
|
|
@ -63,6 +66,7 @@ pub enum ModelFamily {
|
|||
Meta,
|
||||
Mistral,
|
||||
/// Nothing records which family wrote the line.
|
||||
#[default]
|
||||
Unrecorded,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,73 @@ pub struct PacketDraft {
|
|||
pub struct DraftError {
|
||||
pub line: usize,
|
||||
pub column: usize,
|
||||
/// What kind of reading failure, from serde's own fixed vocabulary. It
|
||||
/// never holds a token of the answer.
|
||||
pub class: DraftClass,
|
||||
}
|
||||
|
||||
/// Why an answer is not a draft: a closed set of classes, never the offending
|
||||
/// text, so a record may keep it for a held-out item.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum DraftClass {
|
||||
/// Not JSON.
|
||||
Syntax,
|
||||
/// JSON that ends early.
|
||||
Eof,
|
||||
MissingField,
|
||||
UnknownField,
|
||||
/// A value of the wrong JSON type, such as a bare integer for an id.
|
||||
InvalidType,
|
||||
/// A value of the right type that is not one of the field's, such as a
|
||||
/// number out of range.
|
||||
InvalidValue,
|
||||
/// A string that is not the form its field names: a node id spelled
|
||||
/// `entity:x`, an id of the wrong kind ([`crate::form::FormError`]).
|
||||
BadForm,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl DraftClass {
|
||||
/// The code a round record keeps.
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
DraftClass::Syntax => "syntax",
|
||||
DraftClass::Eof => "eof",
|
||||
DraftClass::MissingField => "missing-field",
|
||||
DraftClass::UnknownField => "unknown-field",
|
||||
DraftClass::InvalidType => "invalid-type",
|
||||
DraftClass::InvalidValue => "invalid-value",
|
||||
DraftClass::BadForm => "bad-form",
|
||||
DraftClass::Other => "other",
|
||||
}
|
||||
}
|
||||
|
||||
fn of(e: &serde_json::Error) -> Self {
|
||||
use serde_json::error::Category;
|
||||
match e.classify() {
|
||||
Category::Syntax => DraftClass::Syntax,
|
||||
Category::Eof => DraftClass::Eof,
|
||||
Category::Io => DraftClass::Other,
|
||||
// Only the leading words of serde's message are read, and only to
|
||||
// pick one of these classes; the message is not kept.
|
||||
Category::Data => {
|
||||
let msg = e.to_string();
|
||||
if msg.starts_with("missing field") {
|
||||
DraftClass::MissingField
|
||||
} else if msg.starts_with("unknown field") {
|
||||
DraftClass::UnknownField
|
||||
} else if msg.starts_with("invalid type") {
|
||||
DraftClass::InvalidType
|
||||
} else if msg.starts_with("invalid value") {
|
||||
DraftClass::InvalidValue
|
||||
} else if msg.starts_with("not a ") {
|
||||
DraftClass::BadForm
|
||||
} else {
|
||||
DraftClass::Other
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for DraftError {
|
||||
|
|
@ -72,6 +139,7 @@ fn from_json<T: DeserializeOwned>(answer: &str) -> Result<T, DraftError> {
|
|||
serde_json::from_str(answer.trim()).map_err(|e| DraftError {
|
||||
line: e.line(),
|
||||
column: e.column(),
|
||||
class: DraftClass::of(&e),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -250,6 +318,31 @@ mod tests {
|
|||
CorpusItemId::parse("item:daodejing:1").unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() {
|
||||
let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class;
|
||||
assert_eq!(class("not json"), DraftClass::Syntax);
|
||||
assert_eq!(class("{\"referents\": ["), DraftClass::Eof);
|
||||
assert_eq!(class("{}"), DraftClass::MissingField);
|
||||
// The classes serde's own messages open with, read off a plain error.
|
||||
let of = |r: Result<u8, serde_json::Error>| DraftClass::of(&r.unwrap_err());
|
||||
assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue);
|
||||
assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType);
|
||||
// Every code is distinct and none can hold answer text: they are fixed.
|
||||
let all = [
|
||||
DraftClass::Syntax,
|
||||
DraftClass::Eof,
|
||||
DraftClass::MissingField,
|
||||
DraftClass::UnknownField,
|
||||
DraftClass::InvalidType,
|
||||
DraftClass::InvalidValue,
|
||||
DraftClass::BadForm,
|
||||
DraftClass::Other,
|
||||
];
|
||||
let codes: std::collections::BTreeSet<_> = all.iter().map(|c| c.code()).collect();
|
||||
assert_eq!(codes.len(), all.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_are_pinned_digests_of_their_inputs() {
|
||||
let encoder = encoder_id("fake-model", "abcd");
|
||||
|
|
|
|||
|
|
@ -92,6 +92,10 @@ enum LoopCommand {
|
|||
/// rounds only) or tool-less-relay. Pinned in the run's record.
|
||||
#[arg(long)]
|
||||
lowerer_reach: LowererReach,
|
||||
/// The family of the answering model; rounds count the corpus items
|
||||
/// that family wrote. Defaults to unrecorded.
|
||||
#[arg(long, default_value = "unrecorded")]
|
||||
answerer_family: ModelFamilyArg,
|
||||
},
|
||||
/// Run one round of the run at ROOT. The stores are read only from the
|
||||
/// run's record: the global --corpus, --inventory and --entities are not
|
||||
|
|
@ -744,6 +748,7 @@ fn main() -> Result<()> {
|
|||
answerer,
|
||||
embed_model,
|
||||
lowerer_reach,
|
||||
answerer_family,
|
||||
}) => {
|
||||
let run = RunStores::create(
|
||||
&root,
|
||||
|
|
@ -756,6 +761,7 @@ fn main() -> Result<()> {
|
|||
&answerer,
|
||||
&embed_model,
|
||||
lowerer_reach,
|
||||
answerer_family.into(),
|
||||
)?;
|
||||
println!("opened run:{}", run.record().label);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ use schemars::JsonSchema;
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems};
|
||||
use crate::corpus::{CorpusItem, CorpusItemId, Source};
|
||||
use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source};
|
||||
use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id};
|
||||
use crate::form::{is_attestation, is_slug};
|
||||
use crate::model::{ModelPort, PortError};
|
||||
|
|
@ -107,6 +107,12 @@ pub struct RunRecord {
|
|||
/// How the run's lowering prompts are answered, set once at open.
|
||||
#[cultcache(key = 12)]
|
||||
pub lowerer_reach: LowererReach,
|
||||
/// The family of the model answering the run's encode and lowering
|
||||
/// prompts, set once at open; `Unrecorded` when the opener did not say
|
||||
/// and for a run opened before this was kept.
|
||||
#[cultcache(key = 13, default)]
|
||||
#[schemars(default)]
|
||||
pub answerer_family: ModelFamily,
|
||||
}
|
||||
|
||||
/// How a run's lowering prompts reach the model that answers them (rulings
|
||||
|
|
@ -224,6 +230,8 @@ pub struct RefusalCounts {
|
|||
/// inventory_digest: String::new(),
|
||||
/// cycle: None,
|
||||
/// admit_rules: vec![],
|
||||
/// parse_reasons: vec![],
|
||||
/// own_family: 0,
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
|
|
@ -276,9 +284,22 @@ pub struct RoundRecord {
|
|||
cycle: Option<u32>,
|
||||
/// The rules admission refused items under, in rule order; every Train
|
||||
/// and Validation item refused at `Stage::Admit` is counted once per rule.
|
||||
#[cultcache(key = 18)]
|
||||
#[cultcache(key = 18, default)]
|
||||
#[schemars(default)]
|
||||
admit_rules: Vec<RuleCount>,
|
||||
/// Why items were refused at `Stage::Parse`, in code order, over Train and
|
||||
/// Validation: a [`DraftClass`] code (`rule`), or `unreadable` for an
|
||||
/// item with no answer that read as text. Never a token of an answer.
|
||||
#[cultcache(key = 19, default)]
|
||||
#[schemars(default)]
|
||||
parse_reasons: Vec<RuleCount>,
|
||||
/// Offered (live Train and Validation) `ModelWritten` items whose writer
|
||||
/// family is the run's answerer family. The corpus rule binds
|
||||
/// model judges; the scorer is deterministic, so nothing is refused for
|
||||
/// this: the round labels it.
|
||||
#[cultcache(key = 20, default)]
|
||||
#[schemars(default)]
|
||||
own_family: u32,
|
||||
}
|
||||
|
||||
impl RoundRecord {
|
||||
|
|
@ -336,6 +357,12 @@ impl RoundRecord {
|
|||
pub fn admit_rules(&self) -> &[RuleCount] {
|
||||
&self.admit_rules
|
||||
}
|
||||
pub fn parse_reasons(&self) -> &[RuleCount] {
|
||||
&self.parse_reasons
|
||||
}
|
||||
pub fn own_family(&self) -> u32 {
|
||||
self.own_family
|
||||
}
|
||||
|
||||
/// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split,
|
||||
/// `scored` those that lowered; a mean counts a refused item as 0.
|
||||
|
|
@ -369,16 +396,24 @@ impl RoundRecord {
|
|||
x.admit,
|
||||
x.lower,
|
||||
);
|
||||
if self.admit_rules.is_empty() {
|
||||
line
|
||||
} else {
|
||||
let rules: Vec<String> = self
|
||||
.admit_rules
|
||||
let codes = |counts: &[RuleCount]| {
|
||||
counts
|
||||
.iter()
|
||||
.map(|r| format!("{}x{}", r.rule, r.count))
|
||||
.collect();
|
||||
format!("{line} admit-rules={}", rules.join(","))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
};
|
||||
let mut line = line;
|
||||
if !self.parse_reasons.is_empty() {
|
||||
line = format!("{line} parse-reasons={}", codes(&self.parse_reasons));
|
||||
}
|
||||
if !self.admit_rules.is_empty() {
|
||||
line = format!("{line} admit-rules={}", codes(&self.admit_rules));
|
||||
}
|
||||
if self.own_family > 0 {
|
||||
line = format!("{line} own-family {}", self.own_family);
|
||||
}
|
||||
line
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -550,13 +585,15 @@ impl RunStores {
|
|||
model: &str,
|
||||
embedder: &str,
|
||||
reach: LowererReach,
|
||||
answerer_family: ModelFamily,
|
||||
) -> Result<Self, RunError> {
|
||||
Self::create_between(root, label, paths, model, embedder, reach, || {})
|
||||
Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {})
|
||||
}
|
||||
|
||||
/// `create` with `between` run after the digests are taken and before the
|
||||
/// stores are opened: the seam that lets a test change a store in that window.
|
||||
#[doc(hidden)]
|
||||
#[allow(clippy::too_many_arguments)] // `create`'s six plus the test seam
|
||||
pub fn create_between(
|
||||
root: &Path,
|
||||
label: &str,
|
||||
|
|
@ -564,6 +601,7 @@ impl RunStores {
|
|||
model: &str,
|
||||
embedder: &str,
|
||||
reach: LowererReach,
|
||||
answerer_family: ModelFamily,
|
||||
between: impl FnOnce(),
|
||||
) -> Result<Self, RunError> {
|
||||
if !is_slug(label) {
|
||||
|
|
@ -592,6 +630,7 @@ impl RunStores {
|
|||
chrf_version: CHRF_VERSION.to_string(),
|
||||
inventory_digest: String::new(),
|
||||
lowerer_reach: reach,
|
||||
answerer_family,
|
||||
};
|
||||
// The run works on its own copy; the caller's store is never opened again.
|
||||
let head = root.join(INVENTORY_FILE);
|
||||
|
|
@ -1024,13 +1063,21 @@ fn score_round<M: ModelPort, E: Embedder>(
|
|||
let mut items: Vec<CorpusItem> = Vec::new();
|
||||
let mut fates: Vec<Fate> = Vec::new();
|
||||
let mut admit_rules: BTreeMap<String, u32> = BTreeMap::new();
|
||||
let mut parse_reasons: BTreeMap<String, u32> = BTreeMap::new();
|
||||
// Phase 1 ends at admission: each admitted view waits for its lowering.
|
||||
let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new();
|
||||
for (key, answer) in &answers {
|
||||
let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?;
|
||||
let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?;
|
||||
let fate = match answer.as_deref().map(PacketDraft::parse) {
|
||||
None | Some(Err(_)) => Fate::Refused(Stage::Parse, None),
|
||||
None => {
|
||||
*parse_reasons.entry("unreadable".to_string()).or_default() += 1;
|
||||
Fate::Refused(Stage::Parse, None)
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
*parse_reasons.entry(e.class.code().to_string()).or_default() += 1;
|
||||
Fate::Refused(Stage::Parse, None)
|
||||
}
|
||||
Some(Ok(draft)) => {
|
||||
let packet = draft.into_packet(&id, rev, &encoder);
|
||||
let packet_key = packet.id.to_string();
|
||||
|
|
@ -1172,6 +1219,15 @@ fn score_round<M: ModelPort, E: Embedder>(
|
|||
}
|
||||
}
|
||||
|
||||
let own_family = match run.record.answerer_family {
|
||||
ModelFamily::Unrecorded => 0,
|
||||
family => items
|
||||
.iter()
|
||||
.filter(|item| {
|
||||
matches!(&item.source, Source::ModelWritten { family: f, .. } if *f == family)
|
||||
})
|
||||
.count() as u32,
|
||||
};
|
||||
let n = run.rounds().map_err(store)?.len() as u32 + 1;
|
||||
let record = RoundRecord {
|
||||
n,
|
||||
|
|
@ -1195,6 +1251,11 @@ fn score_round<M: ModelPort, E: Embedder>(
|
|||
.into_iter()
|
||||
.map(|(rule, count)| RuleCount { rule, count })
|
||||
.collect(),
|
||||
parse_reasons: parse_reasons
|
||||
.into_iter()
|
||||
.map(|(rule, count)| RuleCount { rule, count })
|
||||
.collect(),
|
||||
own_family,
|
||||
};
|
||||
Ok(RoundOutcome::Recorded(record))
|
||||
}
|
||||
|
|
@ -1628,6 +1689,8 @@ mod tests {
|
|||
inventory_digest: String::new(),
|
||||
cycle: None,
|
||||
admit_rules: vec![],
|
||||
parse_reasons: vec![],
|
||||
own_family: 0,
|
||||
};
|
||||
assert_eq!(r.n(), 1);
|
||||
}
|
||||
|
|
@ -1844,6 +1907,8 @@ mod tests {
|
|||
inventory_digest: digest.to_string(),
|
||||
cycle: None,
|
||||
admit_rules: vec![],
|
||||
parse_reasons: vec![],
|
||||
own_family: 0,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1861,6 +1926,7 @@ mod tests {
|
|||
chrf_version: String::new(),
|
||||
inventory_digest: digest.to_string(),
|
||||
lowerer_reach: LowererReach::ToolLessRelay,
|
||||
answerer_family: ModelFamily::Unrecorded,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2059,7 +2125,7 @@ mod tests {
|
|||
entities: &entities,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", crate::run::LowererReach::ToolLessRelay,
|
||||
"bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
.unwrap();
|
||||
let mut model = fake();
|
||||
|
|
|
|||
|
|
@ -1346,6 +1346,8 @@ fn loop_open_and_unanswered_round() {
|
|||
"fake-model",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
"--answerer-family",
|
||||
"open-ai",
|
||||
],
|
||||
)
|
||||
};
|
||||
|
|
@ -1395,7 +1397,7 @@ fn loop_open_and_unanswered_round() {
|
|||
assert!(second.status.success(), "{}", out(&second));
|
||||
assert_eq!(
|
||||
out(&second),
|
||||
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 lowering=tool-less-relay\n"
|
||||
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n"
|
||||
);
|
||||
|
||||
// The global store flags are not read by loop round: a drifted store is refused.
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
|
|||
entities: &dir.join("entities.cc"),
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
|
|||
|
||||
use common::{ent, run_over, world};
|
||||
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
|
||||
use weksa::corpus::ModelFamily;
|
||||
use weksa::run::{DriftOf, RunError, RunPaths, RunRecord, RunStores};
|
||||
use weksa::split::fnv1a64;
|
||||
|
||||
|
|
@ -47,7 +48,7 @@ fn a_run_pins_its_stores_and_refuses_drift() {
|
|||
entities: &e,
|
||||
},
|
||||
model,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
};
|
||||
|
||||
|
|
@ -148,7 +149,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() {
|
|||
entities: &e,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -223,7 +224,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() {
|
|||
entities: &dao.entities,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
};
|
||||
|
||||
|
|
@ -268,7 +269,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() {
|
|||
entities,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
};
|
||||
// A directory opens for reading and fails mid-copy, after the destination exists.
|
||||
|
|
@ -381,14 +382,14 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() {
|
|||
entities: &e,
|
||||
};
|
||||
assert_eq!(
|
||||
RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || more(&mut w)).err(),
|
||||
RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(),
|
||||
Some(RunError::Drift(DriftOf::Corpus))
|
||||
);
|
||||
assert!(!root.join("inventory.cc").exists());
|
||||
// Not the copy and not the lock the store's open left beside it.
|
||||
assert!(!root.join("inventory.cc.lock").exists());
|
||||
assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun));
|
||||
RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay).unwrap();
|
||||
RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap();
|
||||
}
|
||||
|
||||
/// Runs `change` in the window between the pins and the open, at `open` and at
|
||||
|
|
@ -411,7 +412,7 @@ fn window(
|
|||
inventory: &i,
|
||||
entities: &e,
|
||||
};
|
||||
let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || {
|
||||
let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || {
|
||||
change(&mut w, &root)
|
||||
})
|
||||
.err();
|
||||
|
|
@ -828,7 +829,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao {
|
|||
entities: &entities,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
.unwrap();
|
||||
Dao {
|
||||
|
|
@ -1477,15 +1478,15 @@ fn errors_echo_no_input() {
|
|||
inventory: &missing,
|
||||
entities: &missing,
|
||||
};
|
||||
let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay)
|
||||
let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
|
||||
.err()
|
||||
.unwrap();
|
||||
note(&e, &e);
|
||||
let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay)
|
||||
let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
|
||||
.err()
|
||||
.unwrap();
|
||||
note(&e, &e);
|
||||
let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay)
|
||||
let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded)
|
||||
.err()
|
||||
.unwrap();
|
||||
note(&e, &e);
|
||||
|
|
@ -1519,7 +1520,7 @@ fn errors_echo_no_input() {
|
|||
entities: &dao.entities,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
.unwrap();
|
||||
let mut model = cycle_fake(&format!("{{\"{canary}\":1}}"));
|
||||
|
|
@ -2449,7 +2450,7 @@ fn a_changed_head_is_drift() {
|
|||
entities: &dao.entities,
|
||||
},
|
||||
"fake-model",
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay,
|
||||
"bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
|
|
@ -2768,7 +2769,7 @@ fn a_summary_line_says_how_lowerings_were_answered() {
|
|||
inventory: &dao.inventory,
|
||||
entities: &dao.entities,
|
||||
};
|
||||
RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent)
|
||||
RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded)
|
||||
.unwrap();
|
||||
let run = RunStores::open(&root).unwrap();
|
||||
assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent);
|
||||
|
|
@ -2947,6 +2948,282 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() {
|
|||
assert!(!dao.run.round_line(&r).contains("admit-rules"));
|
||||
}
|
||||
|
||||
/// Item 1's good draft as JSON, changed by `change`, as the model's answer.
|
||||
fn mangled(change: impl FnOnce(&mut serde_json::Value)) -> String {
|
||||
let mut v: serde_json::Value = serde_json::from_str(&json(&draft_of(1))).unwrap();
|
||||
change(&mut v);
|
||||
v.to_string()
|
||||
}
|
||||
|
||||
/// The round record of a Dao run whose item 1 is answered with `answer` and
|
||||
/// the rest well.
|
||||
fn round_with_item_1(answer: String) -> (Dao, RoundRecord) {
|
||||
let mut dao = dao();
|
||||
let mut answers = good_answers();
|
||||
answers[0].1 = answer;
|
||||
let mut model = fake(&answers);
|
||||
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
(dao, r)
|
||||
}
|
||||
|
||||
fn reasons(r: &RoundRecord) -> Vec<(&str, u32)> {
|
||||
r.parse_reasons()
|
||||
.iter()
|
||||
.map(|c| (c.rule.as_str(), c.count))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_reasons_by_class() {
|
||||
let good = json(&draft_of(1));
|
||||
let truncated = good[..good.len() / 2].to_string();
|
||||
let cases: Vec<(&str, String)> = vec![
|
||||
("syntax", "not json at all".to_string()),
|
||||
("eof", truncated),
|
||||
(
|
||||
"invalid-type",
|
||||
mangled(|v| v["referents"][0]["concept"] = serde_json::json!(7)),
|
||||
),
|
||||
(
|
||||
"bad-form",
|
||||
mangled(|v| v["referents"][0]["concept"] = serde_json::json!("entity:x")),
|
||||
),
|
||||
(
|
||||
"unknown-field",
|
||||
mangled(|v| v["surprise"] = serde_json::json!(1)),
|
||||
),
|
||||
(
|
||||
"missing-field",
|
||||
mangled(|v| {
|
||||
v.as_object_mut().unwrap().remove("referents");
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (code, answer) in cases {
|
||||
let (dao, r) = round_with_item_1(answer);
|
||||
assert_eq!(reasons(&r), [(code, 1)], "{code}");
|
||||
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}");
|
||||
let line = dao.run.round_line(&r);
|
||||
assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}");
|
||||
}
|
||||
// A round nothing refused at Parse carries and prints none.
|
||||
let (dao, r) = round_with_item_1(good);
|
||||
assert!(r.parse_reasons().is_empty());
|
||||
assert!(!dao.run.round_line(&r).contains("parse-reasons"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() {
|
||||
let mut dao = dao();
|
||||
let expected = prompts(&dao.run);
|
||||
let answers = keyed(&good_answers());
|
||||
let round1 = dao.root.join("round-1");
|
||||
std::fs::create_dir_all(&round1).unwrap();
|
||||
for (key, p) in &expected {
|
||||
let bytes = match key.as_str() {
|
||||
// Train: not UTF-8, so no answer parsed from the file.
|
||||
"item:daodejing:1" => vec![0xff, 0xfe],
|
||||
// Validation: not JSON.
|
||||
"item:daodejing:2" => b"nope".to_vec(),
|
||||
_ => answers[key].clone().into_bytes(),
|
||||
};
|
||||
std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap();
|
||||
}
|
||||
let table = table_english();
|
||||
for (key, p) in lowering_prompts(&dao, dao.run.inventory())
|
||||
.iter()
|
||||
.filter(|(key, _)| key == "item:daodejing:3")
|
||||
{
|
||||
let answer = serde_json::json!({ "text": table[key] }).to_string();
|
||||
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
|
||||
}
|
||||
let mut files = AnswerFiles::new(round1, "fake-model".into());
|
||||
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
|
||||
assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_reasons_never_echo_answer() {
|
||||
let canary = "CANARY-7f3a-held-out";
|
||||
// The canary sits where serde would quote it: an unknown field name, an
|
||||
// invalid value, and the raw text of a non-JSON answer.
|
||||
let answers = [
|
||||
mangled(|v| v[canary] = serde_json::json!(1)),
|
||||
mangled(|v| v["referents"][0]["concept"] = serde_json::json!(canary)),
|
||||
format!("{canary} is not json"),
|
||||
];
|
||||
for answer in answers {
|
||||
// Item 2 is Validation: the held-out item carries the canary.
|
||||
let mut dao = dao();
|
||||
let mut a = good_answers();
|
||||
a[1].1 = answer;
|
||||
let mut model = fake(&a);
|
||||
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
assert_eq!(r.validation_refused().parse, 1);
|
||||
let line = dao.run.round_line(&r);
|
||||
assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}");
|
||||
let bytes = std::fs::read(dao.root.join("run.cc")).unwrap();
|
||||
assert!(!bytes.windows(6).any(|w| w == b"CANARY"));
|
||||
assert!(!format!("{r:?}").contains("CANARY"));
|
||||
}
|
||||
}
|
||||
|
||||
/// A run record as written before `answerer_family` (key 13) existed.
|
||||
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.loop_run")]
|
||||
struct PreFamilyRun {
|
||||
#[cultcache(key = 0)]
|
||||
label: String,
|
||||
#[cultcache(key = 1)]
|
||||
corpus: String,
|
||||
#[cultcache(key = 2)]
|
||||
inventory: String,
|
||||
#[cultcache(key = 3)]
|
||||
entities: String,
|
||||
#[cultcache(key = 4)]
|
||||
corpus_digest: String,
|
||||
#[cultcache(key = 5)]
|
||||
entity_digest: String,
|
||||
#[cultcache(key = 6)]
|
||||
prompt_digest: String,
|
||||
#[cultcache(key = 8)]
|
||||
model: String,
|
||||
#[cultcache(key = 9)]
|
||||
embedder: String,
|
||||
#[cultcache(key = 10)]
|
||||
chrf_version: String,
|
||||
#[cultcache(key = 11)]
|
||||
inventory_digest: String,
|
||||
#[cultcache(key = 12)]
|
||||
lowerer_reach: weksa::run::LowererReach,
|
||||
}
|
||||
|
||||
/// A round record as written before `parse_reasons` and `own_family`, and
|
||||
/// before `admit_rules` had a default.
|
||||
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
|
||||
#[cultcache(type = "weksa.loop_round")]
|
||||
struct LegacyRound {
|
||||
#[cultcache(key = 0)]
|
||||
n: u32,
|
||||
#[cultcache(key = 1)]
|
||||
inventory_rev: u32,
|
||||
#[cultcache(key = 2)]
|
||||
prompt_digest: String,
|
||||
#[cultcache(key = 4)]
|
||||
corpus_digest: String,
|
||||
#[cultcache(key = 5)]
|
||||
entity_digest: String,
|
||||
#[cultcache(key = 6)]
|
||||
model: String,
|
||||
#[cultcache(key = 7)]
|
||||
embedder: String,
|
||||
#[cultcache(key = 8)]
|
||||
chrf_version: String,
|
||||
#[cultcache(key = 9)]
|
||||
calls: Vec<weksa::run::Call>,
|
||||
#[cultcache(key = 10)]
|
||||
train: Vec<weksa::run::TrainRow>,
|
||||
#[cultcache(key = 11)]
|
||||
train_mean: weksa::score::SplitMean,
|
||||
#[cultcache(key = 12)]
|
||||
validation_mean: weksa::score::SplitMean,
|
||||
#[cultcache(key = 13)]
|
||||
validation_refused: weksa::run::RefusalCounts,
|
||||
#[cultcache(key = 14)]
|
||||
train_scored: u32,
|
||||
#[cultcache(key = 15)]
|
||||
validation_scored: u32,
|
||||
#[cultcache(key = 16)]
|
||||
inventory_digest: String,
|
||||
#[cultcache(key = 17)]
|
||||
cycle: Option<u32>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_from_before_the_new_fields_decode_with_defaults() {
|
||||
let mut dao = dao();
|
||||
let mut model = fake(&good_answers());
|
||||
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
let rec = dao.run.record().clone();
|
||||
let root = dao.root.clone();
|
||||
let legacy = LegacyRound {
|
||||
n: r.n(),
|
||||
inventory_rev: r.inventory_rev(),
|
||||
prompt_digest: r.prompt_digest().into(),
|
||||
corpus_digest: r.corpus_digest().into(),
|
||||
entity_digest: r.entity_digest().into(),
|
||||
model: r.model().into(),
|
||||
embedder: r.embedder().into(),
|
||||
chrf_version: r.chrf_version().into(),
|
||||
calls: r.calls().to_vec(),
|
||||
train: r.train().to_vec(),
|
||||
train_mean: r.train_mean().clone(),
|
||||
validation_mean: r.validation_mean().clone(),
|
||||
validation_refused: r.validation_refused(),
|
||||
train_scored: r.train_scored(),
|
||||
validation_scored: r.validation_scored(),
|
||||
inventory_digest: r.inventory_digest().into(),
|
||||
cycle: r.cycle(),
|
||||
};
|
||||
let run = PreFamilyRun {
|
||||
label: rec.label.clone(),
|
||||
corpus: rec.corpus.clone(),
|
||||
inventory: rec.inventory.clone(),
|
||||
entities: rec.entities.clone(),
|
||||
corpus_digest: rec.corpus_digest.clone(),
|
||||
entity_digest: rec.entity_digest.clone(),
|
||||
prompt_digest: rec.prompt_digest.clone(),
|
||||
model: rec.model.clone(),
|
||||
embedder: rec.embedder.clone(),
|
||||
chrf_version: rec.chrf_version.clone(),
|
||||
inventory_digest: rec.inventory_digest.clone(),
|
||||
lowerer_reach: rec.lowerer_reach,
|
||||
};
|
||||
drop(dao.run);
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<PreFamilyRun>().unwrap();
|
||||
cache.register_entry_type::<LegacyRound>().unwrap();
|
||||
cache
|
||||
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
|
||||
.unwrap();
|
||||
cache.pull_all_backing_stores().unwrap();
|
||||
cache.put("run:dao".to_string(), &run).unwrap();
|
||||
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
|
||||
drop(cache);
|
||||
|
||||
let reopened = RunStores::open(&root).unwrap();
|
||||
assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded);
|
||||
let rounds = reopened.rounds().unwrap();
|
||||
assert_eq!(rounds.len(), 1);
|
||||
assert_eq!(rounds[0].n(), 1);
|
||||
assert!(rounds[0].admit_rules().is_empty());
|
||||
assert!(rounds[0].parse_reasons().is_empty());
|
||||
assert_eq!(rounds[0].own_family(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_published_run_and_round_schemas_require_none_of_the_new_fields() {
|
||||
let published = weksa::schema::published();
|
||||
let schema = |needle: &str| -> serde_json::Value {
|
||||
let (_, json) = published
|
||||
.iter()
|
||||
.find(|(name, _)| name.contains(needle))
|
||||
.unwrap();
|
||||
serde_json::from_str(json).unwrap()
|
||||
};
|
||||
for (needle, fields) in [
|
||||
("loop_round", &["parse_reasons", "own_family"][..]),
|
||||
("loop_run", &["answerer_family"][..]),
|
||||
] {
|
||||
let s = schema(needle);
|
||||
let required = s["required"].as_array().unwrap();
|
||||
for f in fields {
|
||||
assert!(s["properties"][f].is_object(), "{needle} {f}");
|
||||
assert!(!required.iter().any(|r| r == f), "{needle} {f} is required");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Answers a recall probe from the corpus: the item's own second half, a
|
||||
/// fixed wrong text, or junk.
|
||||
struct Reciter {
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
|
|||
},
|
||||
"m",
|
||||
"bag",
|
||||
weksa::run::LowererReach::BriefedSubagent,
|
||||
weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded
|
||||
)
|
||||
.unwrap();
|
||||
let mut model = Unrelated(vec![]);
|
||||
|
|
@ -112,3 +112,79 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() {
|
|||
assert!(model.0.iter().all(|t| t.contains("work: not published")));
|
||||
assert!(model.0.iter().all(|t| !t.contains("translator:")));
|
||||
}
|
||||
|
||||
/// Answers every prompt with text that is not a draft: nothing is admitted, so
|
||||
/// the round measures only what is counted before admission.
|
||||
struct Nope;
|
||||
|
||||
impl weksa::model::ModelPort for Nope {
|
||||
fn model(&self) -> &str {
|
||||
"nope"
|
||||
}
|
||||
fn answer(
|
||||
&mut self,
|
||||
_: &weksa::context::Prompt,
|
||||
) -> Result<String, weksa::model::PortError> {
|
||||
Ok("not a draft".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
struct Bag;
|
||||
|
||||
impl weksa::score::Embedder for Bag {
|
||||
fn model(&self) -> &str {
|
||||
"bag"
|
||||
}
|
||||
fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
|
||||
Ok(texts.iter().map(|_| vec![1.0]).collect())
|
||||
}
|
||||
}
|
||||
|
||||
fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::RoundRecord) {
|
||||
use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round};
|
||||
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original");
|
||||
let tmp = tempfile::tempdir().unwrap().keep();
|
||||
for f in ["corpus.cc", "entity.cc", "inventory.cc"] {
|
||||
std::fs::copy(src.join(f), tmp.join(f)).unwrap();
|
||||
}
|
||||
let mut run = RunStores::create(
|
||||
&tmp.join("run"),
|
||||
"orig",
|
||||
RunPaths {
|
||||
corpus: &tmp.join("corpus.cc"),
|
||||
inventory: &tmp.join("inventory.cc"),
|
||||
entities: &tmp.join("entity.cc"),
|
||||
},
|
||||
"nope",
|
||||
"bag",
|
||||
LowererReach::ToolLessRelay,
|
||||
family,
|
||||
)
|
||||
.unwrap();
|
||||
let scorer = weksa::score::Scorer::new(Bag);
|
||||
match round(&mut run, &mut Nope, &scorer).unwrap() {
|
||||
RoundOutcome::Recorded(r) => (run, r),
|
||||
RoundOutcome::Unanswered { .. } => panic!("answered"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn own_family_counts_and_labels() {
|
||||
// The original corpus is six Anthropic-written items, all offered.
|
||||
let (run, r) = original_round(ModelFamily::Anthropic);
|
||||
assert_eq!(r.own_family(), 6);
|
||||
assert_eq!(
|
||||
r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
|
||||
6,
|
||||
"every item was refused at Parse and says why"
|
||||
);
|
||||
let line = run.round_line(&r);
|
||||
assert!(line.contains(" own-family 6"), "{line}");
|
||||
|
||||
// A different answerer family, or none recorded, labels nothing.
|
||||
for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] {
|
||||
let (run, r) = original_round(family);
|
||||
assert_eq!(r.own_family(), 0);
|
||||
assert!(!run.round_line(&r).contains("own-family"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -218,7 +218,7 @@
|
|||
]
|
||||
},
|
||||
{
|
||||
"description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.",
|
||||
"description": "Labelled model output; never in the test split. The rule that it is\nnever judged by its own family binds model judges and the split; a\ndeterministic scorer refuses nothing, so the loop labels (the round\ncounts items of the answerer's family) and does not forbid.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ModelWritten": {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "RoundRecord",
|
||||
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```",
|
||||
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n }\n}\n```",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"admit_rules": {
|
||||
|
|
@ -57,6 +57,21 @@
|
|||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"own_family": {
|
||||
"description": "Offered (live Train and Validation) `ModelWritten` items whose writer\nfamily is the run's answerer family. The corpus rule binds\nmodel judges; the scorer is deterministic, so nothing is refused for\nthis: the round labels it.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"default": 0,
|
||||
"minimum": 0
|
||||
},
|
||||
"parse_reasons": {
|
||||
"description": "Why items were refused at `Stage::Parse`, in code order, over Train and\nValidation: a [`DraftClass`] code (`rule`), or `unreadable` for an\nitem with no answer that read as text. Never a token of an answer.",
|
||||
"type": "array",
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "#/$defs/RuleCount"
|
||||
}
|
||||
},
|
||||
"prompt_digest": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@
|
|||
"description": "What a run pinned at open. Key `run:<label>`.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"answerer_family": {
|
||||
"description": "The family of the model answering the run's encode and lowering\nprompts, set once at open; `Unrecorded` when the opener did not say\nand for a run opened before this was kept.",
|
||||
"$ref": "#/$defs/ModelFamily",
|
||||
"default": "Unrecorded"
|
||||
},
|
||||
"chrf_version": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -77,6 +82,27 @@
|
|||
"const": "ToolLessRelay"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ModelFamily": {
|
||||
"description": "The family of the model that wrote a line.",
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Anthropic",
|
||||
"OpenAi",
|
||||
"Google",
|
||||
"Alibaba",
|
||||
"Meta",
|
||||
"Mistral"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Nothing records which family wrote the line.",
|
||||
"type": "string",
|
||||
"const": "Unrecorded"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue