Context builder and checkpoint-only Test readings (cut loop-context-builder) #25
11 changed files with 1057 additions and 95 deletions
232
crates/weksa/src/context.rs
Normal file
232
crates/weksa/src/context.rs
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
//! The one place prompt text is built from corpus text. Corpus text reaches a
|
||||
//! model call only as a [`Prompt`], and a `Prompt` is made only by minting from
|
||||
//! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only
|
||||
//! minting and per-item encoding (leak-owned-by-flow-and-scorer-3,
|
||||
//! loop-context-builder-2). Instruction wording is not written here; a caller
|
||||
//! passes `&'static str`, so the templates are fixed at compile time and hence
|
||||
//! within a run.
|
||||
|
||||
use crate::corpus::CorpusItem;
|
||||
use crate::packet::EntityId;
|
||||
use crate::split::{Checkpoint, Split, SplitScheme};
|
||||
use crate::store::{CorpusStore, normalized};
|
||||
|
||||
/// The live Train items of a corpus, in key order. The only input minting
|
||||
/// reads; it cannot be built from a slice, and no accessor returns text or an
|
||||
/// item.
|
||||
///
|
||||
/// ```
|
||||
/// let dir = tempfile::tempdir().unwrap();
|
||||
/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap();
|
||||
/// assert!(t.is_empty());
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let t = weksa::context::TrainItems { items: vec![] };
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let dir = tempfile::tempdir().unwrap();
|
||||
/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap();
|
||||
/// let _u = t.clone();
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn s<T: serde::Serialize>(_: &T) {}
|
||||
/// let dir = tempfile::tempdir().unwrap();
|
||||
/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap();
|
||||
/// s(&t);
|
||||
/// ```
|
||||
pub struct TrainItems {
|
||||
items: Vec<CorpusItem>,
|
||||
}
|
||||
|
||||
impl TrainItems {
|
||||
pub fn from_corpus(corpus: &CorpusStore) -> anyhow::Result<Self> {
|
||||
let items = corpus
|
||||
.all()?
|
||||
.into_iter()
|
||||
.filter(|item| SplitScheme::V1.live(item) == Some(Split::Train))
|
||||
.collect();
|
||||
Ok(Self { items })
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.items.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.items.is_empty()
|
||||
}
|
||||
|
||||
/// Whether a live Train item of `group` holds `text`, compared by the
|
||||
/// store's own text key.
|
||||
fn holds(&self, group: &str, text: &str) -> bool {
|
||||
let key = normalized(text);
|
||||
self.items
|
||||
.iter()
|
||||
.any(|item| item.group == group && normalized(&item.text) == key)
|
||||
}
|
||||
|
||||
pub fn keys(&self) -> impl Iterator<Item = &str> {
|
||||
self.items.iter().map(|item| item.id.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Why an item is not offered to the encoder.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum OfferRefusal {
|
||||
Withdrawn,
|
||||
/// A model-written item of a Test group is in no split.
|
||||
NoSplit,
|
||||
/// A Test item is offered only under a [`Checkpoint`].
|
||||
TestBeforeCheckpoint,
|
||||
}
|
||||
|
||||
/// One item the encoder may be shown, with the entities a packet for it may
|
||||
/// cite. Built per item, so a call sees one item and nothing of its group.
|
||||
pub struct EncoderOffer {
|
||||
item: CorpusItem,
|
||||
entities: Vec<EntityId>,
|
||||
}
|
||||
|
||||
impl EncoderOffer {
|
||||
/// A round offers Train and Validation items.
|
||||
pub fn for_round(item: &CorpusItem) -> Result<Self, OfferRefusal> {
|
||||
match Self::gate(item)? {
|
||||
Split::Test => Err(OfferRefusal::TestBeforeCheckpoint),
|
||||
_ => Ok(Self::of(item)),
|
||||
}
|
||||
}
|
||||
|
||||
/// A convergence checkpoint offers any live item, Test included.
|
||||
pub fn for_checkpoint(
|
||||
item: &CorpusItem,
|
||||
_checkpoint: &Checkpoint,
|
||||
) -> Result<Self, OfferRefusal> {
|
||||
Self::gate(item)?;
|
||||
Ok(Self::of(item))
|
||||
}
|
||||
|
||||
fn gate(item: &CorpusItem) -> Result<Split, OfferRefusal> {
|
||||
if item.withdrawn {
|
||||
return Err(OfferRefusal::Withdrawn);
|
||||
}
|
||||
SplitScheme::V1.live(item).ok_or(OfferRefusal::NoSplit)
|
||||
}
|
||||
|
||||
fn of(item: &CorpusItem) -> Self {
|
||||
let mut entities: Vec<EntityId> = item.cited_entities().cloned().collect();
|
||||
entities.sort();
|
||||
entities.dedup();
|
||||
Self {
|
||||
item: item.clone(),
|
||||
entities,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn key(&self) -> &str {
|
||||
&self.item.id
|
||||
}
|
||||
|
||||
/// The item's own cited entities, sorted, without repeats.
|
||||
pub fn entities(&self) -> &[EntityId] {
|
||||
&self.entities
|
||||
}
|
||||
}
|
||||
|
||||
/// Text for a model call, and the keys of the items it holds. The only
|
||||
/// constructors are [`Prompt::mint`] and [`Prompt::encode`].
|
||||
///
|
||||
/// ```
|
||||
/// let dir = tempfile::tempdir().unwrap();
|
||||
/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap();
|
||||
/// let p = weksa::context::Prompt::mint("Mint.", &t);
|
||||
/// assert_eq!(p.text(), "Mint.");
|
||||
/// ```
|
||||
///
|
||||
/// Instructions are compile-time text:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let dir = tempfile::tempdir().unwrap();
|
||||
/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap();
|
||||
/// let p = weksa::context::Prompt::mint(&String::from("x"), &t);
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn s<T: serde::Serialize>(_: &T) {}
|
||||
/// let dir = tempfile::tempdir().unwrap();
|
||||
/// let store = weksa::CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
/// let t = weksa::context::TrainItems::from_corpus(&store).unwrap();
|
||||
/// let p = weksa::context::Prompt::mint("x", &t);
|
||||
/// s(&p);
|
||||
/// ```
|
||||
pub struct Prompt {
|
||||
text: String,
|
||||
items: Vec<String>,
|
||||
}
|
||||
|
||||
/// What a prompt prints of an item: its line, and its prior turns only as the
|
||||
/// one owner allows. A prior turn is a copy of text, not a reference, so it can
|
||||
/// outlive a withdrawal or quote a held-out line; its text is printed only when
|
||||
/// a live Train item of the item's group holds it, and the speaker alone is
|
||||
/// printed otherwise. Mint prints no prior turns (`prior: None`).
|
||||
fn block(item: &CorpusItem, prior: Option<&TrainItems>) -> String {
|
||||
let mut lines: Vec<String> = prior
|
||||
.into_iter()
|
||||
.flat_map(|train| {
|
||||
item.prior_turns.iter().map(move |t| {
|
||||
if train.holds(&item.group, &t.text) {
|
||||
format!("{}: {}", t.speaker, t.text)
|
||||
} else {
|
||||
format!("{}: {ELIDED}", t.speaker)
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
lines.push(format!("[{}] {}: {}", item.id, item.speaker, item.text));
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
/// Printed in place of a prior turn whose text no live Train item holds.
|
||||
const ELIDED: &str = "[elided]";
|
||||
|
||||
impl Prompt {
|
||||
/// Instructions, then every Train item in key order.
|
||||
pub fn mint(instructions: &'static str, train: &TrainItems) -> Prompt {
|
||||
let mut parts = vec![instructions.to_string()];
|
||||
parts.extend(train.items.iter().map(|item| block(item, None)));
|
||||
Prompt {
|
||||
text: parts.join("\n\n"),
|
||||
items: train.keys().map(str::to_string).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Instructions, the one offered item with its prior turns resolved
|
||||
/// against `train`, and the entities a packet for it may cite.
|
||||
pub fn encode(instructions: &'static str, offer: &EncoderOffer, train: &TrainItems) -> Prompt {
|
||||
let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect();
|
||||
Prompt {
|
||||
text: format!(
|
||||
"{instructions}\n\n{}\n\nentities: {}",
|
||||
block(&offer.item, Some(train)),
|
||||
ids.join(", ")
|
||||
),
|
||||
items: vec![offer.item.id.clone()],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn text(&self) -> &str {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// The keys of the items the text holds; what a round record lists.
|
||||
pub fn item_keys(&self) -> &[String] {
|
||||
&self.items
|
||||
}
|
||||
}
|
||||
|
|
@ -97,6 +97,20 @@ pub struct CorpusItem {
|
|||
pub names: Vec<NameSpan>,
|
||||
}
|
||||
|
||||
impl CorpusItem {
|
||||
/// The entities a packet for this item may cite (leak-owned-by-flow-and-scorer-3
|
||||
/// (3)): the entities its text names, then its speaker, then its prior-turn
|
||||
/// speakers (which may repeat). `PacketStore::admit` and `EncoderOffer` read
|
||||
/// only this.
|
||||
pub fn cited_entities(&self) -> impl Iterator<Item = &EntityId> {
|
||||
self.names
|
||||
.iter()
|
||||
.map(|n| &n.entity)
|
||||
.chain(std::iter::once(&self.speaker))
|
||||
.chain(self.prior_turns.iter().map(|t| &t.speaker))
|
||||
}
|
||||
}
|
||||
|
||||
/// A byte range `[start, end)` of an item's text that names an entity. The
|
||||
/// store admits it only when both ends sit on extended grapheme cluster
|
||||
/// boundaries (UAX #29) of the NFC text and the range's bytes equal one of the
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
//! train/validation/test split, the stores that admit them, and the scorer
|
||||
//! (chrF, meaning cosine, leak diagnostics).
|
||||
|
||||
pub mod context;
|
||||
pub mod corpus;
|
||||
pub mod daodejing;
|
||||
pub mod entity;
|
||||
|
|
|
|||
|
|
@ -56,12 +56,6 @@ pub struct SplitLeak {
|
|||
pub worst: Option<WorstItem>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct LeakDiagnostic {
|
||||
pub validation: SplitLeak,
|
||||
pub test: SplitLeak,
|
||||
}
|
||||
|
||||
/// The distinct word n-grams of a text for each order 1..=ORDERS, over its
|
||||
/// normalised word segments.
|
||||
fn grams(text: &str) -> Vec<HashSet<String>> {
|
||||
|
|
@ -145,25 +139,29 @@ fn split_leak(
|
|||
leak
|
||||
}
|
||||
|
||||
/// Reads minted text (inventory glosses, option labels, later ontology text)
|
||||
/// against the held-out items. Per held-out item and for n in 1..=3, its word
|
||||
/// n-grams that occur in no Train item are its novel n-grams; coverage at each
|
||||
/// n is the covered over the novel n-grams summed across the split's items
|
||||
/// (unigrams over all minted strings, longer grams within one string). Honest
|
||||
/// single-word glosses saturate n = 1, so the leak signature is pooled n >= 2
|
||||
/// and the worst item at n >= 2. Train items are not measured; items in no
|
||||
/// split are dropped.
|
||||
pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> LeakDiagnostic {
|
||||
/// The held-out split `want` read against minted text: Train items are the
|
||||
/// live Train items; the readings are over the live items of `want`.
|
||||
pub(super) fn held_out(minted: &[&str], items: &[CorpusItem], want: Split) -> SplitLeak {
|
||||
let of = |want: Split| -> Vec<&CorpusItem> {
|
||||
items
|
||||
.iter()
|
||||
.filter(|i| SplitScheme::V1.split(i) == Some(want))
|
||||
.filter(|i| SplitScheme::V1.live(i) == Some(want))
|
||||
.collect()
|
||||
};
|
||||
let train = union(of(Split::Train).iter().map(|i| i.text.as_str()));
|
||||
let minted_grams = union(minted.iter().copied());
|
||||
LeakDiagnostic {
|
||||
validation: split_leak(minted, &train, &minted_grams, &of(Split::Validation)),
|
||||
test: split_leak(minted, &train, &minted_grams, &of(Split::Test)),
|
||||
}
|
||||
split_leak(minted, &train, &minted_grams, &of(want))
|
||||
}
|
||||
|
||||
/// Reads minted text (inventory glosses, option labels, later ontology text)
|
||||
/// against the Validation items. Per Validation item and for n in 1..=3, its
|
||||
/// word n-grams that occur in no Train item are its novel n-grams; coverage at
|
||||
/// each n is the covered over the novel n-grams summed across the split's
|
||||
/// items (unigrams over all minted strings, longer grams within one string).
|
||||
/// Honest single-word glosses saturate n = 1, so the leak signature is pooled
|
||||
/// n >= 2 and the worst item at n >= 2. Train items are not measured; withdrawn
|
||||
/// items and items in no split are dropped. Test is read only through
|
||||
/// `TestReadings`.
|
||||
pub fn leak_diagnostic(minted: &[&str], items: &[CorpusItem]) -> SplitLeak {
|
||||
held_out(minted, items, Split::Validation)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
//! output against the whole reference. Nothing is deleted, hidden or replaced
|
||||
//! before either is read, and the scorer takes no input from a packet, so no
|
||||
//! text the output carries can raise a score except by resembling the
|
||||
//! reference. No door judges text; the diagnostics report.
|
||||
//! reference. No door judges text; the diagnostics report. The round path
|
||||
//! (`aggregate`, `leak_diagnostic`) reads Train and Validation; Test is read
|
||||
//! only through `TestReadings::at_checkpoint` (convergence-test).
|
||||
|
||||
mod chrf;
|
||||
mod diagnostics;
|
||||
|
|
@ -12,13 +14,11 @@ mod words;
|
|||
use anyhow::{Result, ensure};
|
||||
|
||||
pub use chrf::{CHRF_VERSION, chrf};
|
||||
pub use diagnostics::{
|
||||
LeakDiagnostic, OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic,
|
||||
};
|
||||
pub use diagnostics::{OrderCoverage, PairChrf, SplitLeak, WorstItem, leak_diagnostic};
|
||||
pub use embed::{Embedder, OllamaEmbedder, cosine};
|
||||
|
||||
use crate::corpus::CorpusItem;
|
||||
use crate::split::{Split, SplitScheme};
|
||||
use crate::split::{Checkpoint, Split, SplitScheme};
|
||||
|
||||
/// One reference/output pair; both are scored whole.
|
||||
pub struct ScoreInput<'a> {
|
||||
|
|
@ -111,34 +111,132 @@ pub struct SplitMean {
|
|||
pub meaning: Option<f64>,
|
||||
}
|
||||
|
||||
/// The round readings a round record may hold. The scorer decides when Test
|
||||
/// can be read, by type: `Aggregate` has no Test field.
|
||||
///
|
||||
/// ```
|
||||
/// let agg = weksa::score::aggregate(&[]);
|
||||
/// let _ = agg.validation;
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let agg = weksa::score::aggregate(&[]);
|
||||
/// let _ = agg.test;
|
||||
/// ```
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct Aggregate {
|
||||
pub train: SplitMean,
|
||||
pub validation: SplitMean,
|
||||
pub test: SplitMean,
|
||||
}
|
||||
|
||||
/// Groups scores by the item's derived split. Items in no split are counted
|
||||
/// nowhere. The scorer only groups: the loop decides when Test is read.
|
||||
pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate {
|
||||
let mean = |want: Split| {
|
||||
let of: Vec<&ItemScore> = scored
|
||||
.iter()
|
||||
.filter(|(item, _)| SplitScheme::V1.split(item) == Some(want))
|
||||
.map(|(_, score)| score)
|
||||
.collect();
|
||||
let average = |values: Vec<f64>| {
|
||||
(!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)
|
||||
};
|
||||
SplitMean {
|
||||
count: of.len(),
|
||||
chrf: average(of.iter().map(|s| s.chrf).collect()),
|
||||
meaning: average(of.iter().filter_map(|s| s.meaning).collect()),
|
||||
}
|
||||
/// The mean of the scores of the items read in split `want`.
|
||||
fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean {
|
||||
let of: Vec<&ItemScore> = scored
|
||||
.iter()
|
||||
.filter(|(item, _)| SplitScheme::V1.live(item) == Some(want))
|
||||
.map(|(_, score)| score)
|
||||
.collect();
|
||||
let average = |values: Vec<f64>| {
|
||||
(!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)
|
||||
};
|
||||
Aggregate {
|
||||
train: mean(Split::Train),
|
||||
validation: mean(Split::Validation),
|
||||
test: mean(Split::Test),
|
||||
SplitMean {
|
||||
count: of.len(),
|
||||
chrf: average(of.iter().map(|s| s.chrf).collect()),
|
||||
meaning: average(of.iter().filter_map(|s| s.meaning).collect()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Groups scores by the split each item is read in. Items read in no split
|
||||
/// (withdrawn, or model-written in a Test group) are counted nowhere.
|
||||
pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate {
|
||||
Aggregate {
|
||||
train: split_mean(scored, Split::Train),
|
||||
validation: split_mean(scored, Split::Validation),
|
||||
}
|
||||
}
|
||||
|
||||
/// The Test split's readings, scalars only: no item key, no minted string, no
|
||||
/// pair. The checkpoint decision (convergence-test) needs numbers; a key beside
|
||||
/// a minted string is the handle an edit step would act on. The only producer
|
||||
/// is `at_checkpoint`, which consumes a `Checkpoint`.
|
||||
///
|
||||
/// ```
|
||||
/// use weksa::score::TestReadings;
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// fn s<T: serde::Serialize>(_: &T) {}
|
||||
/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]);
|
||||
/// assert_eq!(t.count(), 0);
|
||||
/// s(&0u8);
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use weksa::score::TestReadings;
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]);
|
||||
/// let _u = t.clone();
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use weksa::score::TestReadings;
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]);
|
||||
/// let _ = format!("{t:?}");
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use weksa::score::TestReadings;
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// fn s<T: serde::Serialize>(_: &T) {}
|
||||
/// let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &[], &[]);
|
||||
/// s(&t);
|
||||
/// ```
|
||||
pub struct TestReadings {
|
||||
mean: SplitMean,
|
||||
leak: SplitLeak,
|
||||
}
|
||||
|
||||
impl TestReadings {
|
||||
/// Reads the live Test items against the scores and the minted strings,
|
||||
/// by the same `split_leak` method the Validation reading uses. The
|
||||
/// checkpoint is spent.
|
||||
pub fn at_checkpoint(
|
||||
checkpoint: Checkpoint,
|
||||
scored: &[(&CorpusItem, ItemScore)],
|
||||
minted: &[&str],
|
||||
items: &[CorpusItem],
|
||||
) -> TestReadings {
|
||||
let _spent = checkpoint;
|
||||
TestReadings {
|
||||
mean: split_mean(scored, Split::Test),
|
||||
leak: diagnostics::held_out(minted, items, Split::Test),
|
||||
}
|
||||
}
|
||||
|
||||
/// Scored Test items.
|
||||
pub fn count(&self) -> usize {
|
||||
self.mean.count
|
||||
}
|
||||
|
||||
pub fn chrf(&self) -> Option<f64> {
|
||||
self.mean.chrf
|
||||
}
|
||||
|
||||
pub fn meaning(&self) -> Option<f64> {
|
||||
self.mean.meaning
|
||||
}
|
||||
|
||||
/// Test items in the leak reading.
|
||||
pub fn items(&self) -> usize {
|
||||
self.leak.items
|
||||
}
|
||||
|
||||
/// The highest chrF of any minted string against any Test item; the score
|
||||
/// only, never the pair.
|
||||
pub fn max_chrf(&self) -> Option<f64> {
|
||||
self.leak.max_chrf.as_ref().map(|p| p.chrf)
|
||||
}
|
||||
|
||||
pub fn coverage_by_n(&self) -> &[OrderCoverage] {
|
||||
&self.leak.coverage_by_n
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
//! The train/validation/test split is derived from an item's group and nothing
|
||||
//! else. No stored assignment exists, so no writer can reassign one.
|
||||
//! else. No stored assignment exists, so no writer can reassign one. An item is
|
||||
//! read in its live split; Test is read only under a Checkpoint.
|
||||
|
||||
use crate::corpus::{CorpusItem, Source};
|
||||
|
||||
|
|
@ -51,4 +52,55 @@ impl SplitScheme {
|
|||
split => Some(split),
|
||||
}
|
||||
}
|
||||
|
||||
/// The split an item is read in; a withdrawn item is read in none.
|
||||
pub fn live(self, item: &CorpusItem) -> Option<Split> {
|
||||
if item.withdrawn {
|
||||
None
|
||||
} else {
|
||||
self.split(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Permission to read Test once. The loop declares one at a convergence
|
||||
/// checkpoint (convergence-test); `TestReadings::at_checkpoint` consumes it, so
|
||||
/// one declaration buys one Test reading, and `EncoderOffer::for_checkpoint`
|
||||
/// borrows it first.
|
||||
///
|
||||
/// ```
|
||||
/// use weksa::score::TestReadings;
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// let c = Checkpoint::declare(3);
|
||||
/// assert_eq!(c.round(), 3);
|
||||
/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]);
|
||||
/// ```
|
||||
///
|
||||
/// One declaration, one reading:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use weksa::score::TestReadings;
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// let c = Checkpoint::declare(3);
|
||||
/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]);
|
||||
/// let _ = TestReadings::at_checkpoint(c, &[], &[], &[]);
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use weksa::split::Checkpoint;
|
||||
/// let c = Checkpoint::declare(3);
|
||||
/// let _again = c.clone();
|
||||
/// ```
|
||||
pub struct Checkpoint {
|
||||
round: u32,
|
||||
}
|
||||
|
||||
impl Checkpoint {
|
||||
pub fn declare(round: u32) -> Self {
|
||||
Self { round }
|
||||
}
|
||||
|
||||
pub fn round(&self) -> u32 {
|
||||
self.round
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -430,7 +430,7 @@ fn check_entities(item: &CorpusItem, entities: &EntityStore) -> Result<(), Store
|
|||
/// (U+0307) after i is dropped, so İx and i̇x collide as the old case fold made
|
||||
/// them; so does plain ix, with both. Over-merging a rare sequence errs toward
|
||||
/// held, a lost collision toward leak. U+0307 after any other letter stays.
|
||||
fn normalized(text: &str) -> String {
|
||||
pub(crate) fn normalized(text: &str) -> String {
|
||||
let mapped: String = text.nfkc().flat_map(fold_variant).collect();
|
||||
// The dot goes before the last NFKC so the marks left on the i compose with
|
||||
// it (i, dot, acute reads as the i with an acute).
|
||||
|
|
@ -958,9 +958,7 @@ impl PacketStore {
|
|||
}
|
||||
.into());
|
||||
}
|
||||
let on_item = item.names.iter().any(|n| &n.entity == entity)
|
||||
|| &item.speaker == entity
|
||||
|| item.prior_turns.iter().any(|t| &t.speaker == entity);
|
||||
let on_item = item.cited_entities().any(|e| e == entity);
|
||||
if !on_item {
|
||||
return Err(StoreRefusal::EntityNotOnItem {
|
||||
entity: entity.clone(),
|
||||
|
|
|
|||
412
crates/weksa/tests/context.rs
Normal file
412
crates/weksa/tests/context.rs
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
mod common;
|
||||
|
||||
use common::{cast, ent, entities_in, group_in_bucket, item_in, model, world};
|
||||
use weksa::context::{EncoderOffer, OfferRefusal, Prompt, TrainItems};
|
||||
use weksa::split::Checkpoint;
|
||||
use weksa::{
|
||||
CorpusItem, CorpusItemId, CorpusStore, NameSpan, PacketId, PacketStore, StoreError,
|
||||
StoreRefusal, Turn,
|
||||
};
|
||||
|
||||
const MINT: &str = "Mint glosses from the lines below.";
|
||||
const ENCODE: &str = "Encode the line below.";
|
||||
|
||||
struct Corpus {
|
||||
_dir: tempfile::TempDir,
|
||||
store: CorpusStore,
|
||||
}
|
||||
|
||||
/// One live item per split, a withdrawn Train item and a model-written
|
||||
/// Test-group item, each with a marker word unique to it.
|
||||
fn corpus() -> Corpus {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let entities = entities_in(dir.path());
|
||||
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let mut put = |item: CorpusItem| {
|
||||
let withdrawn = item.withdrawn;
|
||||
let id = item.id.clone();
|
||||
store
|
||||
.add(
|
||||
CorpusItem {
|
||||
withdrawn: false,
|
||||
..item
|
||||
},
|
||||
&entities,
|
||||
)
|
||||
.unwrap();
|
||||
if withdrawn {
|
||||
store.withdraw(&id).unwrap();
|
||||
}
|
||||
};
|
||||
put(item_in(
|
||||
"item:a:1",
|
||||
"alphamarker speaks",
|
||||
&group_in_bucket(10),
|
||||
cast(),
|
||||
));
|
||||
put(item_in(
|
||||
"item:b:1",
|
||||
"betamarker speaks",
|
||||
&group_in_bucket(50),
|
||||
cast(),
|
||||
));
|
||||
put(item_in(
|
||||
"item:c:1",
|
||||
"gammamarker speaks",
|
||||
&group_in_bucket(90),
|
||||
cast(),
|
||||
));
|
||||
let mut gone = item_in(
|
||||
"item:a:2",
|
||||
"deltamarker speaks",
|
||||
&group_in_bucket(10),
|
||||
cast(),
|
||||
);
|
||||
gone.withdrawn = true;
|
||||
put(gone);
|
||||
put(item_in(
|
||||
"item:c:2",
|
||||
"epsilonmarker speaks",
|
||||
&group_in_bucket(90),
|
||||
model(),
|
||||
));
|
||||
Corpus { _dir: dir, store }
|
||||
}
|
||||
|
||||
fn get(c: &Corpus, key: &str) -> CorpusItem {
|
||||
c.store.get(key).unwrap().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn train_items_hold_only_live_train() {
|
||||
let c = corpus();
|
||||
let t = TrainItems::from_corpus(&c.store).unwrap();
|
||||
assert_eq!(t.keys().collect::<Vec<_>>(), ["item:a:1"]);
|
||||
assert_eq!((t.len(), t.is_empty()), (1, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn train_items_are_in_key_order() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let entities = entities_in(dir.path());
|
||||
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
for (key, text) in [
|
||||
("item:z:1", "zed line"),
|
||||
("item:a:1", "ay line"),
|
||||
("item:m:1", "em line"),
|
||||
] {
|
||||
store
|
||||
.add(item_in(key, text, &group_in_bucket(10), cast()), &entities)
|
||||
.unwrap();
|
||||
}
|
||||
let t = TrainItems::from_corpus(&store).unwrap();
|
||||
assert_eq!(
|
||||
t.keys().collect::<Vec<_>>(),
|
||||
["item:a:1", "item:m:1", "item:z:1"]
|
||||
);
|
||||
assert_eq!((t.len(), t.is_empty()), (3, false));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let none = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let none = TrainItems::from_corpus(&none).unwrap();
|
||||
assert_eq!((none.len(), none.is_empty()), (0, true));
|
||||
assert_eq!(Prompt::mint(MINT, &none).text(), MINT);
|
||||
assert!(Prompt::mint(MINT, &none).item_keys().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mint_prompt_holds_train_text_only() {
|
||||
let c = corpus();
|
||||
let t = TrainItems::from_corpus(&c.store).unwrap();
|
||||
let p = Prompt::mint(MINT, &t);
|
||||
assert!(p.text().starts_with(MINT));
|
||||
assert!(p.text().contains("alphamarker"));
|
||||
for other in ["betamarker", "gammamarker", "deltamarker", "epsilonmarker"] {
|
||||
assert!(!p.text().contains(other), "{other}");
|
||||
}
|
||||
assert_eq!(p.item_keys(), ["item:a:1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompts_render_deterministically() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let entities = entities_in(dir.path());
|
||||
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let mut one = item_in("item:a:1", "Hello there.", &group_in_bucket(10), cast());
|
||||
one.prior_turns = vec![Turn {
|
||||
speaker: common::eid("brokkr"),
|
||||
text: "Well?".into(),
|
||||
}];
|
||||
store.add(one, &entities).unwrap();
|
||||
store
|
||||
.add(
|
||||
item_in("item:a:2", "Second line.", &group_in_bucket(10), cast()),
|
||||
&entities,
|
||||
)
|
||||
.unwrap();
|
||||
let t = TrainItems::from_corpus(&store).unwrap();
|
||||
assert_eq!(
|
||||
Prompt::mint(MINT, &t).text(),
|
||||
format!(
|
||||
"{MINT}\n\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line."
|
||||
)
|
||||
);
|
||||
assert_eq!(Prompt::mint(MINT, &t).item_keys(), ["item:a:1", "item:a:2"]);
|
||||
|
||||
let offer = EncoderOffer::for_round(&store.get("item:a:2").unwrap().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
Prompt::encode(ENCODE, &offer, &t).text(),
|
||||
format!("{ENCODE}\n\n[item:a:2] entity:nibu: Second line.\n\nentities: entity:nibu")
|
||||
);
|
||||
assert_eq!(Prompt::encode(ENCODE, &offer, &t).item_keys(), ["item:a:2"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_offer_split_gate() {
|
||||
let c = corpus();
|
||||
let cp = Checkpoint::declare(1);
|
||||
let train = get(&c, "item:a:1");
|
||||
let validation = get(&c, "item:b:1");
|
||||
let test = get(&c, "item:c:1");
|
||||
let no_split = get(&c, "item:c:2");
|
||||
let withdrawn = get(&c, "item:a:2");
|
||||
assert!(withdrawn.withdrawn);
|
||||
let round = |i: &CorpusItem| EncoderOffer::for_round(i).err();
|
||||
let check = |i: &CorpusItem| EncoderOffer::for_checkpoint(i, &cp).err();
|
||||
|
||||
assert_eq!(round(&train), None);
|
||||
assert_eq!(round(&validation), None);
|
||||
assert_eq!(round(&test), Some(OfferRefusal::TestBeforeCheckpoint));
|
||||
assert_eq!(round(&no_split), Some(OfferRefusal::NoSplit));
|
||||
assert_eq!(round(&withdrawn), Some(OfferRefusal::Withdrawn));
|
||||
|
||||
assert_eq!(check(&train), None);
|
||||
assert_eq!(check(&validation), None);
|
||||
assert_eq!(check(&test), None);
|
||||
assert_eq!(check(&no_split), Some(OfferRefusal::NoSplit));
|
||||
assert_eq!(check(&withdrawn), Some(OfferRefusal::Withdrawn));
|
||||
|
||||
// Withdrawn wins over NoSplit.
|
||||
let mut both = no_split.clone();
|
||||
both.withdrawn = true;
|
||||
assert_eq!(round(&both), Some(OfferRefusal::Withdrawn));
|
||||
assert_eq!(check(&both), Some(OfferRefusal::Withdrawn));
|
||||
// A withdrawn Test item is Withdrawn, not TestBeforeCheckpoint.
|
||||
let mut test_gone = test.clone();
|
||||
test_gone.withdrawn = true;
|
||||
assert_eq!(round(&test_gone), Some(OfferRefusal::Withdrawn));
|
||||
}
|
||||
|
||||
fn span(slug: &str, start: u32, end: u32) -> NameSpan {
|
||||
NameSpan {
|
||||
entity: common::eid(slug),
|
||||
start,
|
||||
end,
|
||||
}
|
||||
}
|
||||
|
||||
/// An item naming kay (e1), spoken by hero (e2) after a turn by brokkr (e3),
|
||||
/// and a sibling of the same group naming someone (e4).
|
||||
fn cited_world() -> (Corpus, CorpusItem, CorpusItem) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut entities = entities_in(dir.path());
|
||||
entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
|
||||
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let group = group_in_bucket(10);
|
||||
let mut mine = item_in("item:a:1", "Kay waits here.", &group, cast());
|
||||
mine.speaker = common::eid("hero");
|
||||
mine.names = vec![span("kay", 0, 3)];
|
||||
mine.prior_turns = vec![
|
||||
Turn {
|
||||
speaker: common::eid("brokkr"),
|
||||
text: "Where?".into(),
|
||||
},
|
||||
Turn {
|
||||
speaker: common::eid("brokkr"),
|
||||
text: "Again?".into(),
|
||||
},
|
||||
];
|
||||
let mut sibling = item_in("item:a:2", "Someone waits there.", &group, cast());
|
||||
sibling.names = vec![span("someone", 0, 7)];
|
||||
store.add(mine.clone(), &entities).unwrap();
|
||||
store.add(sibling.clone(), &entities).unwrap();
|
||||
(Corpus { _dir: dir, store }, mine, sibling)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoder_offer_entities_are_the_items_own() {
|
||||
let (c, mine, sibling) = cited_world();
|
||||
let t = TrainItems::from_corpus(&c.store).unwrap();
|
||||
assert_eq!(
|
||||
mine.cited_entities()
|
||||
.map(|e| e.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"entity:kay",
|
||||
"entity:hero",
|
||||
"entity:brokkr",
|
||||
"entity:brokkr"
|
||||
],
|
||||
"names, speaker, prior-turn speakers, repeats kept"
|
||||
);
|
||||
let offer = EncoderOffer::for_round(&mine).unwrap();
|
||||
let ids: Vec<&str> = offer.entities().iter().map(|e| e.as_str()).collect();
|
||||
assert_eq!(ids, ["entity:brokkr", "entity:hero", "entity:kay"]);
|
||||
assert!(
|
||||
!ids.contains(&"entity:someone"),
|
||||
"a sibling's entity is not the item's"
|
||||
);
|
||||
assert_eq!(offer.key(), "item:a:1");
|
||||
let p = Prompt::encode(ENCODE, &offer, &t);
|
||||
assert!(
|
||||
p.text()
|
||||
.ends_with("entities: entity:brokkr, entity:hero, entity:kay")
|
||||
);
|
||||
assert!(p.text().contains("[item:a:1]"));
|
||||
assert!(!p.text().contains("item:a:2") && !p.text().contains("Someone waits"));
|
||||
assert_eq!(p.item_keys(), ["item:a:1"]);
|
||||
let sib = EncoderOffer::for_round(&sibling).unwrap();
|
||||
assert_eq!(sib.entities().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offer_and_store_agree() {
|
||||
let mut w = world();
|
||||
w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
|
||||
let mut item = item_in(
|
||||
"item:nibu:2",
|
||||
"Kay waits here.",
|
||||
&group_in_bucket(10),
|
||||
cast(),
|
||||
);
|
||||
item.speaker = common::eid("hero");
|
||||
item.names = vec![span("kay", 0, 3)];
|
||||
item.prior_turns = vec![Turn {
|
||||
speaker: common::eid("brokkr"),
|
||||
text: "Where?".into(),
|
||||
}];
|
||||
w.corpus.add(item.clone(), &w.entities).unwrap();
|
||||
let offer = EncoderOffer::for_round(&item).unwrap();
|
||||
|
||||
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||
let all = ["nibu", "hero", "kay", "brokkr", "player", "s", "someone"];
|
||||
for (n, slug) in all.iter().enumerate() {
|
||||
let mut p = common::good();
|
||||
p.id = PacketId::parse(&format!("packet:{n:016x}")).unwrap();
|
||||
p.provenance.corpus_item = CorpusItemId::parse("item:nibu:2").unwrap();
|
||||
p.referents[0].entity = Some(common::eid(slug));
|
||||
let offered = offer.entities().contains(&common::eid(slug));
|
||||
match packets.admit(p, &w.inventory, &w.corpus, &w.entities) {
|
||||
Ok(_) => assert!(offered, "{slug} admitted but not offered"),
|
||||
Err(StoreError::Refused(StoreRefusal::EntityNotOnItem { .. })) => {
|
||||
assert!(!offered, "{slug} offered but refused")
|
||||
}
|
||||
Err(other) => panic!("{slug}: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Items of one Train group and of a Test group, with the lines each prior
|
||||
/// turn quotes.
|
||||
#[test]
|
||||
fn prior_turns_print_only_text_a_live_train_item_holds() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let entities = entities_in(dir.path());
|
||||
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let (train_g, other_train_g) = (group_in_bucket(10), group_in_bucket(11));
|
||||
let turn = |text: &str| Turn {
|
||||
speaker: common::eid("brokkr"),
|
||||
text: text.into(),
|
||||
};
|
||||
let mut add = |key: &str, text: &str, group: &str, turns: Vec<Turn>| {
|
||||
let mut item = item_in(key, text, group, cast());
|
||||
item.prior_turns = turns;
|
||||
store.add(item, &entities).unwrap();
|
||||
};
|
||||
add("item:a:1", "withdrawnsecret line", &train_g, vec![]);
|
||||
add("item:a:4", "livecontext line", &train_g, vec![]);
|
||||
add(
|
||||
"item:a:2",
|
||||
"reply line",
|
||||
&train_g,
|
||||
vec![turn("withdrawnsecret line")],
|
||||
);
|
||||
add(
|
||||
"item:a:3",
|
||||
"second reply",
|
||||
&train_g,
|
||||
vec![turn("testsecret line")],
|
||||
);
|
||||
add(
|
||||
"item:a:5",
|
||||
"third reply",
|
||||
&train_g,
|
||||
vec![turn("livecontext line")],
|
||||
);
|
||||
add(
|
||||
"item:a:6",
|
||||
"other reply",
|
||||
&train_g,
|
||||
vec![turn("elsewhere line")],
|
||||
);
|
||||
add("item:o:1", "elsewhere line", &other_train_g, vec![]);
|
||||
add(
|
||||
"item:b:1",
|
||||
"validation reply",
|
||||
&group_in_bucket(50),
|
||||
vec![turn("testsecret line")],
|
||||
);
|
||||
add("item:c:1", "testsecret line", &group_in_bucket(90), vec![]);
|
||||
store.withdraw("item:a:1").unwrap();
|
||||
|
||||
let t = TrainItems::from_corpus(&store).unwrap();
|
||||
let mint = Prompt::mint(MINT, &t);
|
||||
for secret in ["withdrawnsecret", "testsecret", "validation reply"] {
|
||||
assert!(!mint.text().contains(secret), "mint printed {secret}");
|
||||
}
|
||||
assert!(
|
||||
!mint.text().contains("brokkr"),
|
||||
"mint prints no prior turns"
|
||||
);
|
||||
|
||||
let encode = |key: &str| {
|
||||
let item = store.get(key).unwrap().unwrap();
|
||||
Prompt::encode(ENCODE, &EncoderOffer::for_round(&item).unwrap(), &t)
|
||||
.text()
|
||||
.to_string()
|
||||
};
|
||||
let withdrawn = encode("item:a:2");
|
||||
assert!(!withdrawn.contains("withdrawnsecret"));
|
||||
assert!(withdrawn.contains("entity:brokkr: [elided]\n[item:a:2]"));
|
||||
let test_quote = encode("item:a:3");
|
||||
assert!(!test_quote.contains("testsecret"));
|
||||
assert!(test_quote.contains("entity:brokkr: [elided]"));
|
||||
let validation = encode("item:b:1");
|
||||
assert!(!validation.contains("testsecret"));
|
||||
assert!(validation.contains("entity:brokkr: [elided]"));
|
||||
assert!(encode("item:a:5").contains("entity:brokkr: livecontext line\n[item:a:5]"));
|
||||
let cross_group = encode("item:a:6");
|
||||
assert!(
|
||||
!cross_group.contains("elsewhere"),
|
||||
"a held line of another group is not this group's"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prior_turn_match_uses_the_stores_text_key() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let entities = entities_in(dir.path());
|
||||
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
||||
let g = group_in_bucket(10);
|
||||
store
|
||||
.add(item_in("item:a:1", "Context Line", &g, cast()), &entities)
|
||||
.unwrap();
|
||||
let mut reply = item_in("item:a:2", "reply", &g, cast());
|
||||
reply.prior_turns = vec![Turn {
|
||||
speaker: common::eid("brokkr"),
|
||||
text: "context line".into(),
|
||||
}];
|
||||
store.add(reply.clone(), &entities).unwrap();
|
||||
let t = TrainItems::from_corpus(&store).unwrap();
|
||||
let p = Prompt::encode(ENCODE, &EncoderOffer::for_round(&reply).unwrap(), &t);
|
||||
assert!(p.text().contains("entity:brokkr: context line"));
|
||||
}
|
||||
|
|
@ -10,9 +10,10 @@ use common::{cast, group_in_bucket, item_in, model};
|
|||
use serde_json::Value;
|
||||
use weksa::CorpusItem;
|
||||
use weksa::score::{
|
||||
Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, aggregate, chrf, cosine,
|
||||
Embedder, ItemScore, OllamaEmbedder, ScoreInput, Scorer, TestReadings, aggregate, chrf, cosine,
|
||||
leak_diagnostic,
|
||||
};
|
||||
use weksa::split::Checkpoint;
|
||||
|
||||
/// (reference, hypothesis, sacrebleu 2.4.3 sentence_chrf), map 491536c.
|
||||
const VECTORS: [(&str, &str, f64); 6] = [
|
||||
|
|
@ -353,12 +354,6 @@ fn aggregate_by_derived_split() {
|
|||
assert_eq!(agg.train.chrf, Some(15.0));
|
||||
assert_eq!(agg.train.meaning, Some(0.5), "None is not averaged as zero");
|
||||
assert_eq!((agg.validation.count, agg.validation.chrf), (1, Some(40.0)));
|
||||
assert_eq!((agg.test.count, agg.test.chrf), (1, Some(60.0)));
|
||||
assert_eq!(
|
||||
agg.test.meaning,
|
||||
Some(0.6),
|
||||
"the model-written test item is in no split"
|
||||
);
|
||||
assert_eq!(aggregate(&[]).validation.chrf, None);
|
||||
}
|
||||
|
||||
|
|
@ -385,7 +380,6 @@ fn train(key: &str, text: &str) -> CorpusItem {
|
|||
/// (novel, covered) per order for the Validation split.
|
||||
fn read(minted: &[&str], items: &[CorpusItem]) -> Vec<(usize, usize)> {
|
||||
leak_diagnostic(minted, items)
|
||||
.validation
|
||||
.coverage_by_n
|
||||
.iter()
|
||||
.map(|c| (c.novel, c.covered))
|
||||
|
|
@ -414,7 +408,7 @@ fn leak_diagnostic_catches_chunked_glosses() {
|
|||
test_model,
|
||||
],
|
||||
);
|
||||
let pair = d.validation.max_chrf.as_ref().unwrap();
|
||||
let pair = d.max_chrf.as_ref().unwrap();
|
||||
assert!(
|
||||
pair.chrf < 50.0,
|
||||
"each chunk alone scores low: {}",
|
||||
|
|
@ -426,24 +420,21 @@ fn leak_diagnostic_catches_chunked_glosses() {
|
|||
read(&minted, &[held.clone(), unrelated, train_item.clone()]),
|
||||
[(16, 12), (14, 8), (12, 4)]
|
||||
);
|
||||
let worst = d.validation.worst.as_ref().unwrap();
|
||||
let worst = d.worst.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
(worst.item.as_str(), worst.n, worst.covered, worst.novel),
|
||||
("item:v:1", 2, 8, 11)
|
||||
);
|
||||
assert_eq!(d.validation.items, 2);
|
||||
assert_eq!(d.test.items, 0, "a model-written test item is in no split");
|
||||
assert_eq!(d.test.max_chrf, None);
|
||||
assert_eq!(d.test.worst, None);
|
||||
assert_eq!(d.items, 2);
|
||||
|
||||
let only_train = leak_diagnostic(&minted, &[train_item]);
|
||||
assert_eq!(only_train.validation.items, 0);
|
||||
assert_eq!(only_train.validation.max_chrf, None);
|
||||
assert_eq!(only_train.validation.worst, None);
|
||||
assert_eq!(only_train.items, 0);
|
||||
assert_eq!(only_train.max_chrf, None);
|
||||
assert_eq!(only_train.worst, None);
|
||||
|
||||
let full = leak_diagnostic(&[HELD_OUT], &[held]);
|
||||
assert_eq!(full.validation.coverage_by_n[2].share, Some(1.0));
|
||||
assert!(full.validation.max_chrf.unwrap().chrf > 99.0);
|
||||
assert_eq!(full.coverage_by_n[2].share, Some(1.0));
|
||||
assert!(full.max_chrf.unwrap().chrf > 99.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -485,9 +476,9 @@ fn coverage_reads_joined_labels() {
|
|||
"kick the bucket",
|
||||
] {
|
||||
let d = leak_diagnostic(&[label], &items);
|
||||
let by_n = &d.validation.coverage_by_n;
|
||||
let by_n = &d.coverage_by_n;
|
||||
assert_eq!((by_n[2].novel, by_n[2].covered), (3, 1), "{label}");
|
||||
let worst = d.validation.worst.unwrap();
|
||||
let worst = d.worst.unwrap();
|
||||
assert_eq!((worst.n, worst.covered), (2, 2), "{label}");
|
||||
}
|
||||
}
|
||||
|
|
@ -520,7 +511,7 @@ fn coverage_pooled_per_order() {
|
|||
"left", "group", "found", "place", "storm", "after", "warn", "leave", "harbor",
|
||||
];
|
||||
let d = leak_diagnostic(&glosses, &items);
|
||||
let by_n = &d.validation.coverage_by_n;
|
||||
let by_n = &d.coverage_by_n;
|
||||
assert_eq!(by_n.iter().map(|c| c.n).collect::<Vec<_>>(), [1, 2, 3]);
|
||||
assert_eq!((by_n[0].novel, by_n[0].covered), (6, 5));
|
||||
assert!(by_n[0].share.unwrap() < 1.0);
|
||||
|
|
@ -528,7 +519,7 @@ fn coverage_pooled_per_order() {
|
|||
assert!(c.novel > 0);
|
||||
assert_eq!((c.covered, c.share), (0, Some(0.0)));
|
||||
}
|
||||
assert_eq!(d.validation.worst, None);
|
||||
assert_eq!(d.worst, None);
|
||||
|
||||
// no novel n-gram reads None, never NaN; an empty split reads the same
|
||||
let items = [
|
||||
|
|
@ -536,16 +527,13 @@ fn coverage_pooled_per_order() {
|
|||
validation("item:v:1", "harbors guard the quiet"),
|
||||
];
|
||||
let d = leak_diagnostic(&["anything at all"], &items);
|
||||
for split in [&d.validation, &d.test] {
|
||||
assert_eq!(split.coverage_by_n.len(), 3);
|
||||
assert!(
|
||||
split
|
||||
.coverage_by_n
|
||||
.iter()
|
||||
.all(|c| (c.novel, c.share) == (0, None))
|
||||
);
|
||||
assert_eq!(split.worst, None);
|
||||
}
|
||||
assert_eq!(d.coverage_by_n.len(), 3);
|
||||
assert!(
|
||||
d.coverage_by_n
|
||||
.iter()
|
||||
.all(|c| (c.novel, c.share) == (0, None))
|
||||
);
|
||||
assert_eq!(d.worst, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -569,7 +557,6 @@ fn worst_item_rules() {
|
|||
validation("item:v:2", "delta epsilon"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
|
|
@ -582,7 +569,6 @@ fn worst_item_rules() {
|
|||
validation("item:v:2", "alpha beta gamma"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta", "delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!(w.item, "item:v:1");
|
||||
|
|
@ -592,7 +578,6 @@ fn worst_item_rules() {
|
|||
validation("item:v:2", "alpha beta gamma delta"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta gamma", "delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!((w.item.as_str(), w.covered, w.novel), ("item:v:2", 2, 3));
|
||||
|
|
@ -601,17 +586,13 @@ fn worst_item_rules() {
|
|||
validation("item:v:1", "delta epsilon"),
|
||||
validation("item:v:2", "delta epsilon"),
|
||||
];
|
||||
let w = leak_diagnostic(&["delta epsilon"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
let w = leak_diagnostic(&["delta epsilon"], &items).worst.unwrap();
|
||||
assert_eq!(w.item, "item:v:1");
|
||||
let items = [
|
||||
validation("item:v:1", "alpha beta gamma"),
|
||||
train("item:t:1", "alpha beta"),
|
||||
];
|
||||
let w = leak_diagnostic(&["alpha beta gamma"], &items)
|
||||
.validation
|
||||
.worst
|
||||
.unwrap();
|
||||
assert_eq!((w.n, w.covered, w.novel), (2, 1, 1));
|
||||
|
|
@ -631,7 +612,7 @@ fn leak_diagnostic_reports_the_best_pair_and_the_first_of_ties() {
|
|||
let b = validation("item:v:2", HELD_OUT);
|
||||
// the better minted string comes second; equal items tie, the first is kept
|
||||
let d = leak_diagnostic(&["zzzzzz qqqqqq", HELD_OUT], &[a, b]);
|
||||
let pair = d.validation.max_chrf.unwrap();
|
||||
let pair = d.max_chrf.unwrap();
|
||||
assert_eq!(
|
||||
(pair.minted.as_str(), pair.item.as_str()),
|
||||
(HELD_OUT, "item:v:1")
|
||||
|
|
@ -842,3 +823,149 @@ fn an_unspaced_script_ends_a_word_beside_any_other_script() {
|
|||
assert_eq!(read(&["\u{4e00}\u{301}\u{200d}", "ab"], &items)[0], (2, 2));
|
||||
assert_eq!(read(&["\u{4e00}"], &items)[0], (2, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_path_reads_no_test() {
|
||||
let train = item_in("item:a:1", "t", &group_in_bucket(10), cast());
|
||||
let validation = item_in("item:b:1", "v", &group_in_bucket(50), cast());
|
||||
let mut withdrawn = item_in("item:b:2", "w", &group_in_bucket(50), cast());
|
||||
withdrawn.withdrawn = true;
|
||||
let test = item_in("item:c:1", "x", &group_in_bucket(90), cast());
|
||||
let test_two = item_in("item:c:2", "y", &group_in_bucket(90), cast());
|
||||
let reading = |test_score: f64, extra: bool| {
|
||||
let mut scored = vec![
|
||||
(&train, score(10.0, Some(0.5))),
|
||||
(&validation, score(40.0, Some(0.8))),
|
||||
(&withdrawn, score(99.0, Some(0.99))),
|
||||
(&test, score(test_score, Some(0.1))),
|
||||
];
|
||||
if extra {
|
||||
scored.push((&test_two, score(100.0, Some(1.0))));
|
||||
}
|
||||
aggregate(&scored)
|
||||
};
|
||||
let base = reading(60.0, false);
|
||||
assert_eq!(
|
||||
(base.validation.count, base.validation.chrf),
|
||||
(1, Some(40.0))
|
||||
);
|
||||
assert_eq!(base.validation.meaning, Some(0.8));
|
||||
assert_eq!((base.train.count, base.train.chrf), (1, Some(10.0)));
|
||||
assert_eq!(reading(5.0, false), base, "a Test score moves nothing");
|
||||
assert_eq!(reading(5.0, true), base, "a Test item adds nothing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leak_diagnostic_reads_validation_only() {
|
||||
let test = item_in("item:t:9", HELD_OUT, &group_in_bucket(90), cast());
|
||||
let val = validation("item:v:1", "gardening is quiet work");
|
||||
let mut gone = validation("item:v:2", HELD_OUT);
|
||||
gone.withdrawn = true;
|
||||
let train_item = train("item:t:1", UNRELATED);
|
||||
let items = [test, val.clone(), gone, train_item.clone()];
|
||||
|
||||
let d = leak_diagnostic(&[HELD_OUT], &items);
|
||||
assert_eq!(d.items, 1, "Validation only, withdrawn dropped");
|
||||
assert_ne!(d.max_chrf.as_ref().unwrap().item, "item:t:9");
|
||||
assert_eq!(d.max_chrf.as_ref().unwrap().item, "item:v:1");
|
||||
assert!(d.max_chrf.as_ref().unwrap().chrf < 50.0);
|
||||
assert_eq!(d.worst, None);
|
||||
let without_test = leak_diagnostic(&[HELD_OUT], &[val.clone(), train_item.clone()]);
|
||||
assert_eq!(d, without_test, "removing the Test item changes nothing");
|
||||
|
||||
let on_validation = leak_diagnostic(&["gardening is quiet work"], &items);
|
||||
assert!(on_validation.max_chrf.unwrap().chrf > 99.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_readings_at_checkpoint() {
|
||||
let train_item = train("item:t:1", UNRELATED);
|
||||
let test = item_in("item:c:1", HELD_OUT, &group_in_bucket(90), cast());
|
||||
let test_model = item_in(
|
||||
"item:c:2",
|
||||
"model words here",
|
||||
&group_in_bucket(90),
|
||||
model(),
|
||||
);
|
||||
let mut gone = item_in("item:c:3", "withdrawn words", &group_in_bucket(90), cast());
|
||||
gone.withdrawn = true;
|
||||
let val = validation("item:v:1", "gardening is quiet work");
|
||||
let items = [
|
||||
train_item.clone(),
|
||||
test.clone(),
|
||||
test_model.clone(),
|
||||
gone.clone(),
|
||||
val.clone(),
|
||||
];
|
||||
let scored = [
|
||||
(&train_item, score(10.0, Some(0.5))),
|
||||
(&val, score(40.0, Some(0.8))),
|
||||
(&test, score(60.0, Some(0.6))),
|
||||
(&test_model, score(100.0, Some(1.0))),
|
||||
(&gone, score(90.0, Some(0.9))),
|
||||
];
|
||||
let t = TestReadings::at_checkpoint(Checkpoint::declare(2), &scored, &[HELD_OUT], &items);
|
||||
assert_eq!(
|
||||
(t.count(), t.chrf(), t.meaning()),
|
||||
(1, Some(60.0), Some(0.6))
|
||||
);
|
||||
assert_eq!(
|
||||
t.items(),
|
||||
1,
|
||||
"model-written and withdrawn Test items are excluded"
|
||||
);
|
||||
assert!(t.max_chrf().unwrap() > 99.0);
|
||||
assert_eq!(t.coverage_by_n()[2].share, Some(1.0));
|
||||
|
||||
// A minted string that is not the Test text reads low; the same method as
|
||||
// the Validation reading, so the same minted text reads alike in each.
|
||||
let t = TestReadings::at_checkpoint(Checkpoint::declare(2), &scored, &["zzz"], &items);
|
||||
assert!(t.max_chrf().unwrap() < 20.0);
|
||||
assert_eq!(t.coverage_by_n()[2].share, Some(0.0));
|
||||
|
||||
let empty = TestReadings::at_checkpoint(Checkpoint::declare(2), &[], &[HELD_OUT], &[]);
|
||||
assert_eq!(
|
||||
(empty.count(), empty.chrf(), empty.meaning()),
|
||||
(0, None, None)
|
||||
);
|
||||
assert_eq!((empty.items(), empty.max_chrf()), (0, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validation_and_test_leak_agree_in_method() {
|
||||
// The same text and minted strings, once in Validation and once in Test,
|
||||
// read the same numbers.
|
||||
let minted = chunks(3, " ");
|
||||
let minted = refs(&minted);
|
||||
let train_item = train("item:t:1", UNRELATED);
|
||||
let v = leak_diagnostic(
|
||||
&minted,
|
||||
&[validation("item:v:1", HELD_OUT), train_item.clone()],
|
||||
);
|
||||
let test = item_in("item:c:1", HELD_OUT, &group_in_bucket(90), cast());
|
||||
let t = TestReadings::at_checkpoint(Checkpoint::declare(1), &[], &minted, &[test, train_item]);
|
||||
assert_eq!(t.items(), v.items);
|
||||
assert_eq!(t.max_chrf(), v.max_chrf.as_ref().map(|p| p.chrf));
|
||||
assert_eq!(t.coverage_by_n(), v.coverage_by_n.as_slice());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn held_out_train_union_excludes_withdrawn_train_items() {
|
||||
let mut gone = train("item:t:1", HELD_OUT);
|
||||
gone.withdrawn = true;
|
||||
let held = validation("item:v:1", HELD_OUT);
|
||||
let d = leak_diagnostic(
|
||||
&[HELD_OUT],
|
||||
&[gone, held.clone(), train("item:t:2", UNRELATED)],
|
||||
);
|
||||
assert_eq!(
|
||||
d.coverage_by_n
|
||||
.iter()
|
||||
.map(|c| (c.novel, c.covered))
|
||||
.collect::<Vec<_>>(),
|
||||
[(12, 12), (11, 11), (10, 10)],
|
||||
"a withdrawn Train text makes no n-gram known"
|
||||
);
|
||||
let live = leak_diagnostic(&[HELD_OUT], &[train("item:t:1", HELD_OUT), held]);
|
||||
assert_eq!(live.coverage_by_n[0].novel, 0, "a live Train text does");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -151,3 +151,32 @@ fn published_schemas_match() {
|
|||
assert_eq!(held, json, "{name} drifted; run `weksa schema`");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_is_the_split_of_an_item_that_is_not_withdrawn() {
|
||||
for (bucket, want) in [
|
||||
(10, Split::Train),
|
||||
(50, Split::Validation),
|
||||
(90, Split::Test),
|
||||
] {
|
||||
let group = group_in_bucket(bucket);
|
||||
let live = item(&group, cast());
|
||||
assert_eq!(SplitScheme::V1.live(&live), Some(want));
|
||||
let mut gone = live.clone();
|
||||
gone.withdrawn = true;
|
||||
assert_eq!(
|
||||
SplitScheme::V1.live(&gone),
|
||||
None,
|
||||
"withdrawn: no live split"
|
||||
);
|
||||
assert_eq!(
|
||||
SplitScheme::V1.split(&gone),
|
||||
Some(want),
|
||||
"split still names the group's split"
|
||||
);
|
||||
}
|
||||
let mut model_test = item(&group_in_bucket(90), model());
|
||||
assert_eq!(SplitScheme::V1.live(&model_test), None);
|
||||
model_test.withdrawn = true;
|
||||
assert_eq!(SplitScheme::V1.live(&model_test), None);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
|
|||
2. `packet-store`: Rust crate with packet, inventory and corpus types, the validator, the derived split, the stores and the `weksa` CLI. Landed at `28284cc`.
|
||||
2b. Landed. `packet-store-corpus`: corpus labels tell the truth (`corpus-labels-tell-truth`): `ModelFamily` and the generator string on `ModelWritten`; `item:nibu:1` ModelWritten (Unrecorded); `item:brokkr:1` and its fixture carry `codex-local-source-aware-draft`, family OpenAi; intake refuses aliased keys, a whole item changed under its key, and a held item's text under a new key; corpus.cc reseeded. Speaker state is not in 2b: it lives in Speaker and Scene documents and the packet's stance, not on the item. Waits on 2. (spec)
|
||||
2c. Landed. `entity-store`: `ProperName` and `Referent.name` deleted; `EntityId`, `Referent.entity`, the `weksa.entity` document and its store; corpus name spans and entity-id speakers; the packet store's entity checks; seed entities and spans for the cast items; schema v2. Waits on 2b. (spec r1)
|
||||
3. Landed at `869c82b`. `scorer`: chrF, meaning cosine behind an `Embedder` port (Ollama adapter with `num_thread`), leak diagnostics, aggregation by derived split; library only, `weksa score` comes with the loop. h2 landed the r2 fix pass on its branch; r3 deletes the name mask (the scorer reads output and reference only) and pools novel coverage per order over word keys that split joined labels. Waits on 2c for merge order. (spec r3)
|
||||
3. Landed at `869c82b`. `scorer`: chrF, meaning cosine behind an `Embedder` port (Ollama adapter with `num_thread`), leak diagnostics, aggregation by derived split (Train and Validation only on the round path; Test only through `TestReadings::at_checkpoint`, which consumes a `Checkpoint`); library only, `weksa score` comes with the loop. h2 landed the r2 fix pass on its branch; r3 deletes the name mask (the scorer reads output and reference only) and pools novel coverage per order over word keys that split joined labels. Waits on 2c for merge order. (spec r3)
|
||||
2d. `entity-door-lows`: invisible format characters refused in entity forms (ZWJ, ZWNJ, CGJ and variation selectors kept); the form length bound per question `form-word-bound` (recommended: bytes only); attested origins (`Cast { persona, fixture }`, `Live { persona }`) and `CastAuthored.author`; empty provenance strings refused at the store; S6 under NFKC plus case fold and no-alphanumeric texts refused; `next_key` refuses at `u32::MAX`; CLI flags that do not apply refuse and no error echoes a value. Owns follow-ups `entity-door-lows` and `s6-unicode-normal-form`. (spec r1)
|
||||
3b. `lowerer-first-cut` (follow-up `golf-loop-lowerer-cut`): the section above. `lower`, `weksa lower`, `weksa score`, `weksa packet proof`, inventory rev 2 for `nibu-proof-001`.
|
||||
4. `speaker-state`: packet `stance` (felt affect and restraint, slot `Stance`, packet schema v4); formality, warmth and intensity out of the seed and the standard; only `PacketStore::admit` yields a `LoweringInput`, and it refuses packets for withdrawn items (follow-up `lowering-input-from-packet-store`). (spec r1)
|
||||
|
|
@ -401,7 +401,8 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
|
|||
4b. `speaker-scene`: slots `Speaker` and `Scene` with their seed features; `weksa.speaker` (`speaker:<entity-slug>@<n>`) and `weksa.scene` (`scene:<group>@<n>`, participants and ordered-pair relations) documents with `Annotator` `Operator` and `Model { family, generator }`, validated by the packet's shared feature check, append-only per key; their stores, `weksa speaker` and `weksa scene` commands, published schemas; Model-annotated cast speakers and scenes for the committed items. Waits on 2d and 4; `pd-speaker-annotation` blocks only the public-domain annotator.
|
||||
5. `language-ontology`: `weksa.community` from `data/cultural-ontology/` (owns follow-up `edda-referent-yaml-parse`), `weksa.ontology_entry` for en-US and ja-JP from `data/target-language-ontology/` (YAML deleted), speaker communities, deterministic `select`, the lowering view (packet without trace, speaker, scene, selected entries). Waits on 4.
|
||||
6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`.
|
||||
7. `golf-loop-en`: the loop driver, the Train-only context builder for inventory- and ontology-minting calls (follow-up `loop-context-builder-train-only`), the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter, encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are `&'static str`), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1)
|
||||
7. `golf-loop-en`: the loop driver, the packet store bound to one inventory store (`packet-store-inventory-binding`), `ModelPort` with the `claude -p` adapter (it takes only `&Prompt`), encoder (packet with stance) and lowerer prompts, round records, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3.
|
||||
9. `ja-panel`: ja-JP into the loop's panel. Waits on 7.
|
||||
10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`.
|
||||
|
|
|
|||
Loading…
Reference in a new issue