Stance ablation beside recall; lowering prompt pinned to the stance; grade scale stated #46
7 changed files with 1083 additions and 12 deletions
|
|
@ -66,7 +66,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab
|
|||
/// Fixed per run by `PROMPT_SOURCE_DIGEST`.
|
||||
pub const LOWER_INSTRUCTIONS: &str = "\
|
||||
Say the packet below in English (en-US), as one utterance of its speaker: all of its meaning and nothing it does not carry.
|
||||
Every id is the inventory's listed below and every feature value is as listed. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says.
|
||||
Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says.
|
||||
A referent with an entity is named by one of that entity's forms on the entities line.
|
||||
Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary.";
|
||||
|
||||
|
|
|
|||
|
|
@ -11,11 +11,14 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
|
|||
use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
|
||||
use weksa::inventory::{proof_revision, seed};
|
||||
use weksa::packet::proof_packet;
|
||||
use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round};
|
||||
use weksa::model::RecordedAnswers;
|
||||
use weksa::run::{
|
||||
AblationOutcome, CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
|
||||
};
|
||||
use weksa::score::ScoreInput;
|
||||
use weksa::{
|
||||
AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore,
|
||||
Scorer, SplitScheme, StoreError, lower, pronounce, validate,
|
||||
Scorer, Split, SplitScheme, StoreError, lower, pronounce, validate,
|
||||
};
|
||||
|
||||
#[derive(Parser)]
|
||||
|
|
@ -112,6 +115,19 @@ enum LoopCommand {
|
|||
/// half). Prompts and answers live in `<ROOT>/recall`; the command exits 3
|
||||
/// until every prompt has an answer. A diagnostic, never a gate.
|
||||
Recall { root: PathBuf },
|
||||
/// Ask whether the lowerer uses a packet's stance: for each live Train and
|
||||
/// Validation packet of round ROUND, ask again with the stance as it is,
|
||||
/// emptied, and swapped for the next item's, and read how far the lowering
|
||||
/// moves from the round's own answer. Prompts and answers live in
|
||||
/// `<ROOT>/ablation-<ROUND>`; the command exits 3 until every prompt has an
|
||||
/// answer. A diagnostic, never a gate; Test is never touched.
|
||||
Ablate {
|
||||
root: PathBuf,
|
||||
#[arg(long)]
|
||||
round: u32,
|
||||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
/// Run one edit-accept cycle of the run at ROOT. The stores are read only
|
||||
/// from the run's record. The editor's prompt, then the candidate round's
|
||||
/// encode prompts, then its lowering prompts are written in turn; the
|
||||
|
|
@ -785,6 +801,36 @@ fn main() -> Result<()> {
|
|||
RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)),
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Ablate {
|
||||
root,
|
||||
round,
|
||||
ollama_url,
|
||||
}) => {
|
||||
let run = RunStores::open(&root)?;
|
||||
let model = run.record().model.clone();
|
||||
let mut recorded =
|
||||
RecordedAnswers::new(root.join(format!("round-{round}")), model.clone());
|
||||
let mut files = AnswerFiles::new(run.ablation_dir(round), model);
|
||||
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
|
||||
match run.stance_ablation(round, &mut recorded, &mut files, &scorer)? {
|
||||
AblationOutcome::Unanswered { prompts } => {
|
||||
for digest in &prompts {
|
||||
println!("unanswered {digest}");
|
||||
}
|
||||
println!("unanswered {} prompts", prompts.len());
|
||||
std::process::exit(3);
|
||||
}
|
||||
AblationOutcome::Read(readings) => {
|
||||
for (split, reading) in readings {
|
||||
let name = match split {
|
||||
Split::Train => "train",
|
||||
_ => "validation",
|
||||
};
|
||||
println!("{name} {}", reading.line());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Recall { root }) => {
|
||||
let run = RunStores::open(&root)?;
|
||||
let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone());
|
||||
|
|
|
|||
|
|
@ -48,6 +48,37 @@ impl AnswerFiles {
|
|||
}
|
||||
}
|
||||
|
||||
/// Answers read from one directory and nothing else: `<digest>.answer` is the
|
||||
/// reply to the prompt whose digest is `<digest>`. A prompt with no file is
|
||||
/// [`PortError::Unanswered`]; this port never creates or writes anything. The
|
||||
/// one reader of answer files: [`AnswerFiles`] reads through it.
|
||||
pub struct RecordedAnswers {
|
||||
dir: PathBuf,
|
||||
model: String,
|
||||
}
|
||||
|
||||
impl RecordedAnswers {
|
||||
pub fn new(dir: PathBuf, model: String) -> Self {
|
||||
Self { dir, model }
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelPort for RecordedAnswers {
|
||||
fn model(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
|
||||
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
|
||||
match std::fs::read(self.dir.join(format!("{}.answer", prompt.digest()))) {
|
||||
Ok(bytes) => {
|
||||
String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData))
|
||||
}
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => Err(PortError::Unanswered),
|
||||
Err(e) => Err(PortError::Io(e.kind())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelPort for AnswerFiles {
|
||||
fn model(&self) -> &str {
|
||||
&self.model
|
||||
|
|
@ -56,12 +87,9 @@ impl ModelPort for AnswerFiles {
|
|||
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
|
||||
let io = |e: std::io::Error| PortError::Io(e.kind());
|
||||
let digest = prompt.digest();
|
||||
match std::fs::read(self.dir.join(format!("{digest}.answer"))) {
|
||||
Ok(bytes) => {
|
||||
return String::from_utf8(bytes).map_err(|_| PortError::Io(ErrorKind::InvalidData));
|
||||
}
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => {}
|
||||
Err(e) => return Err(io(e)),
|
||||
match RecordedAnswers::new(self.dir.clone(), self.model.clone()).answer(prompt) {
|
||||
Err(PortError::Unanswered) => {}
|
||||
read => return read,
|
||||
}
|
||||
std::fs::create_dir_all(&self.dir).map_err(io)?;
|
||||
let path = self.dir.join(format!("{digest}.prompt"));
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@
|
|||
//! `ModelPort` (`Prompt::lower`), in two answer phases; the CLI's `lower` still
|
||||
//! uses the template table. The prompt source pin covers `context.rs`,
|
||||
//! `split.rs` and `lower.rs`.
|
||||
//!
|
||||
//! `RunStores::stance_ablation`, like `recall`, is a diagnostic in its own
|
||||
//! directory: it records nothing and never writes its round's directory.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fmt;
|
||||
|
|
@ -39,6 +42,7 @@ use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source};
|
|||
use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id};
|
||||
use crate::form::{is_attestation, is_slug};
|
||||
use crate::model::{ModelPort, PortError};
|
||||
use crate::packet::{Packet, Stance};
|
||||
use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
|
||||
use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
|
||||
use crate::store::{
|
||||
|
|
@ -451,6 +455,9 @@ pub enum RunError {
|
|||
NoFloor,
|
||||
/// A round has no Validation reading.
|
||||
NoValidation,
|
||||
/// The round holds no answer to a lowering prompt the ablation rebuilt: the
|
||||
/// prompt's digest drifted or the head inventory moved since the round.
|
||||
Unrecorded,
|
||||
}
|
||||
|
||||
impl fmt::Display for RunError {
|
||||
|
|
@ -466,6 +473,9 @@ impl fmt::Display for RunError {
|
|||
RunError::NoInventory => f.write_str("the inventory holds no revision"),
|
||||
RunError::Io(kind) => write!(f, "io error {kind:?}"),
|
||||
RunError::Store => f.write_str("a store would not read or write"),
|
||||
RunError::Unrecorded => f.write_str(
|
||||
"the round holds no answer to a lowering prompt rebuilt now: the prompt digest drifted or the head inventory moved since the round",
|
||||
),
|
||||
RunError::Embed => f.write_str("the embedder failed"),
|
||||
RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"),
|
||||
RunError::NoValidation => f.write_str("a reading has no validation item"),
|
||||
|
|
@ -950,6 +960,301 @@ impl RunStores {
|
|||
}
|
||||
}
|
||||
|
||||
/// What the lowerer's answer did to each variant of a packet's stance, per
|
||||
/// split. Means only; no key, text or stance value.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct AblationReading {
|
||||
/// Items whose recorded and replayed lowerings were both usable.
|
||||
pub n: u32,
|
||||
/// Answers dropped as unusable, replay or variant.
|
||||
pub unusable: u32,
|
||||
/// Mean agreement of the replay with the recorded lowering: the noise
|
||||
/// floor a variant is read against.
|
||||
pub replay: Agreement,
|
||||
pub drop: VariantReading,
|
||||
pub permute: VariantReading,
|
||||
}
|
||||
|
||||
/// Means over an empty set are `None`, printed `none`: no measurement, never
|
||||
/// a zero.
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct Agreement {
|
||||
pub chrf: Option<f64>,
|
||||
pub meaning: Option<f64>,
|
||||
}
|
||||
|
||||
/// One variant's reading: agreement with the recorded lowering, pairs whose
|
||||
/// chrF fell below the replay's, and the mean fidelity the item's own stance
|
||||
/// adds (`score(item, replay) - score(item, variant)`).
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct VariantReading {
|
||||
pub n: u32,
|
||||
pub chrf: Option<f64>,
|
||||
pub meaning: Option<f64>,
|
||||
pub moved: u32,
|
||||
pub ref_gain_chrf: Option<f64>,
|
||||
pub ref_gain_meaning: Option<f64>,
|
||||
}
|
||||
|
||||
fn show(x: Option<f64>, places: usize) -> String {
|
||||
x.map_or("none".to_string(), |x| format!("{x:.places$}"))
|
||||
}
|
||||
|
||||
impl AblationReading {
|
||||
pub fn line(&self) -> String {
|
||||
let variant = |name: &str, v: &VariantReading| {
|
||||
format!(
|
||||
"{name} n={} chrf={} meaning={} moved={} ref_gain_chrf={} ref_gain_meaning={}",
|
||||
v.n,
|
||||
show(v.chrf, 2),
|
||||
show(v.meaning, 3),
|
||||
v.moved,
|
||||
show(v.ref_gain_chrf, 2),
|
||||
show(v.ref_gain_meaning, 3)
|
||||
)
|
||||
};
|
||||
format!(
|
||||
"ablation n={} unusable={} replay chrf={} meaning={} | {} | {}",
|
||||
self.n,
|
||||
self.unusable,
|
||||
show(self.replay.chrf, 2),
|
||||
show(self.replay.meaning, 3),
|
||||
variant("drop", &self.drop),
|
||||
variant("permute", &self.permute),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum AblationOutcome {
|
||||
Unanswered { prompts: Vec<String> },
|
||||
/// Train then Validation.
|
||||
Read(Vec<(Split, AblationReading)>),
|
||||
}
|
||||
|
||||
/// What the port made of one lowering prompt.
|
||||
enum Asked {
|
||||
Text(String),
|
||||
Unusable,
|
||||
Waiting,
|
||||
}
|
||||
|
||||
fn asked(port: &mut dyn ModelPort, prompt: &Prompt) -> Result<Asked, RunError> {
|
||||
match port.answer(prompt) {
|
||||
Ok(a) => Ok(LoweringDraft::parse(&a)
|
||||
.ok()
|
||||
.filter(LoweringDraft::usable)
|
||||
.map_or(Asked::Unusable, |d| Asked::Text(d.text))),
|
||||
Err(PortError::Io(ErrorKind::InvalidData)) => Ok(Asked::Unusable),
|
||||
Err(PortError::Unanswered) => Ok(Asked::Waiting),
|
||||
Err(PortError::Io(kind)) => Err(RunError::Io(kind)),
|
||||
}
|
||||
}
|
||||
|
||||
fn mean(xs: impl Iterator<Item = f64>) -> Option<f64> {
|
||||
let (sum, n) = xs.fold((0.0, 0u32), |(s, n), x| (s + x, n + 1));
|
||||
(n > 0).then(|| sum / f64::from(n))
|
||||
}
|
||||
|
||||
type Scored = (ItemScore, ItemScore);
|
||||
|
||||
/// One item's reference text, recorded lowering, replay, and drop and permute
|
||||
/// lowerings.
|
||||
type Lowered = (String, String, String, [Option<String>; 2]);
|
||||
|
||||
impl RunStores {
|
||||
/// `<root>/ablation-<n>`, where a stance ablation's prompts, answers and
|
||||
/// variant packet stores live.
|
||||
pub fn ablation_dir(&self, n: u32) -> PathBuf {
|
||||
self.root.join(format!("ablation-{n}"))
|
||||
}
|
||||
|
||||
/// A diagnostic like [`recall`](Self::recall), never a gate: for each live
|
||||
/// Train and Validation packet of plain round `n`, asks the lowerer again
|
||||
/// (`model`, the replay), with the packet's stance emptied (drop) and with
|
||||
/// the next item's stance in its split (permute), and reads how far the
|
||||
/// lowering moves against the round's own answer (`recorded`). Test and
|
||||
/// withdrawn items are never asked. Records nothing and writes nothing in
|
||||
/// `round-<n>`; the variant stores are rebuilt each call. An answer that
|
||||
/// is not a usable lowering drops its pair and is counted in `unusable`.
|
||||
pub fn stance_ablation<E: Embedder>(
|
||||
&self,
|
||||
n: u32,
|
||||
recorded: &mut dyn ModelPort,
|
||||
model: &mut dyn ModelPort,
|
||||
scorer: &Scorer<E>,
|
||||
) -> Result<AblationOutcome, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
check_drift(self, model, scorer)?;
|
||||
self.rounds()
|
||||
.map_err(store)?
|
||||
.into_iter()
|
||||
.find(|r| r.n() == n && r.cycle().is_none())
|
||||
.ok_or(RunError::Malformed("round"))?;
|
||||
let inventory = self.inventory.inventory().map_err(store)?;
|
||||
|
||||
let dir = self.ablation_dir(n);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?;
|
||||
for name in ["base", "drop", "permute"] {
|
||||
for suffix in ["cc", "cc.lock"] {
|
||||
let _ = std::fs::remove_file(dir.join(format!("{name}.{suffix}")));
|
||||
}
|
||||
}
|
||||
let mut stores = [
|
||||
PacketStore::open(&dir.join("base.cc")).map_err(store)?,
|
||||
PacketStore::open(&dir.join("drop.cc")).map_err(store)?,
|
||||
PacketStore::open(&dir.join("permute.cc")).map_err(store)?,
|
||||
];
|
||||
|
||||
// Live Train and Validation packets, in item key order within a split.
|
||||
let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc"))
|
||||
.map_err(store)?
|
||||
.all()
|
||||
.map_err(store)?;
|
||||
let mut splits: [(Split, Vec<(CorpusItem, Packet)>); 2] =
|
||||
[(Split::Train, Vec::new()), (Split::Validation, Vec::new())];
|
||||
for packet in round_packets {
|
||||
let item = self
|
||||
.corpus
|
||||
.get(packet.provenance.corpus_item.as_str())
|
||||
.map_err(store)?
|
||||
.ok_or(RunError::Store)?;
|
||||
for (split, held) in &mut splits {
|
||||
if SplitScheme::V1.live(&item) == Some(*split) {
|
||||
held.push((item.clone(), packet.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut waiting: Vec<String> = Vec::new();
|
||||
let mut readings = Vec::new();
|
||||
// Texts per split: (item text, recorded, replay, [drop, permute]).
|
||||
let mut texts: Vec<Vec<Lowered>> = Vec::new();
|
||||
let mut unusable = [0u32; 2];
|
||||
for (at, (_, held)) in splits.iter_mut().enumerate() {
|
||||
held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str()));
|
||||
let mut rows = Vec::new();
|
||||
for (i, (item, packet)) in held.iter().enumerate() {
|
||||
let packet_item = packet.provenance.corpus_item.clone();
|
||||
let mut prompt_of = |slot: usize, packet: Packet| -> Result<Prompt, RunError> {
|
||||
let view = stores[slot]
|
||||
.admit(packet, &self.inventory, &self.corpus, &self.entities)
|
||||
.map_err(|_| RunError::Store)?;
|
||||
Prompt::lower(&view, &packet_item, &inventory, &self.entities)
|
||||
.map_err(|_| RunError::Store)
|
||||
};
|
||||
let base = prompt_of(0, packet.clone())?;
|
||||
let mut variants: [Option<Prompt>; 2] = [None, None];
|
||||
if !packet.stance.features.is_empty() {
|
||||
let mut p = packet.clone();
|
||||
p.stance = Stance::default();
|
||||
variants[0] = Some(prompt_of(1, p)?);
|
||||
}
|
||||
if held.len() > 1 {
|
||||
let mut p = packet.clone();
|
||||
p.stance = held[(i + 1) % held.len()].1.stance.clone();
|
||||
let prompt = prompt_of(2, p)?;
|
||||
if prompt.digest() != base.digest() {
|
||||
variants[1] = Some(prompt);
|
||||
}
|
||||
}
|
||||
// The round's own answer must exist; a missing one is not a gap to ask for.
|
||||
let recorded_text = match asked(recorded, &base)? {
|
||||
Asked::Waiting => return Err(RunError::Unrecorded),
|
||||
other => other,
|
||||
};
|
||||
let replay = asked(model, &base)?;
|
||||
let mut got: [Option<String>; 2] = [None, None];
|
||||
for (slot, prompt) in got.iter_mut().zip(&variants) {
|
||||
let Some(prompt) = prompt else { continue };
|
||||
match asked(model, prompt)? {
|
||||
Asked::Text(t) => *slot = Some(t),
|
||||
Asked::Unusable => unusable[at] += 1,
|
||||
Asked::Waiting => {
|
||||
let d = prompt.digest();
|
||||
if !waiting.contains(&d) {
|
||||
waiting.push(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
match (recorded_text, replay) {
|
||||
(Asked::Text(b), Asked::Text(r)) => {
|
||||
rows.push((item.text.clone(), b, r, got));
|
||||
}
|
||||
(_, Asked::Waiting) => {
|
||||
let d = base.digest();
|
||||
if !waiting.contains(&d) {
|
||||
waiting.push(d);
|
||||
}
|
||||
}
|
||||
_ => unusable[at] += 1,
|
||||
}
|
||||
}
|
||||
texts.push(rows);
|
||||
}
|
||||
if !waiting.is_empty() {
|
||||
return Ok(AblationOutcome::Unanswered { prompts: waiting });
|
||||
}
|
||||
|
||||
// One score call: for each item, the replay then each variant, each
|
||||
// against the recorded lowering (agreement) and the item (fidelity).
|
||||
let mut inputs: Vec<ScoreInput> = Vec::new();
|
||||
for (reference, base, replay, got) in texts.iter().flatten() {
|
||||
for output in std::iter::once(replay).chain(got.iter().flatten()) {
|
||||
inputs.push(ScoreInput { reference: base, output });
|
||||
inputs.push(ScoreInput { reference, output });
|
||||
}
|
||||
}
|
||||
let mut scores = if inputs.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
scorer.score(&inputs).map_err(|_| RunError::Embed)?
|
||||
}
|
||||
.into_iter();
|
||||
let mut pair = || -> Result<Scored, RunError> {
|
||||
Ok((scores.next().ok_or(RunError::Store)?, scores.next().ok_or(RunError::Store)?))
|
||||
};
|
||||
for (at, ((split, _), rows)) in splits.iter().zip(&texts).enumerate() {
|
||||
let mut replay: Vec<Scored> = Vec::new();
|
||||
let mut variants: [Vec<(Scored, Scored)>; 2] = [Vec::new(), Vec::new()];
|
||||
for (_, _, _, got) in rows {
|
||||
let r = pair()?;
|
||||
for (k, g) in got.iter().enumerate() {
|
||||
if g.is_some() {
|
||||
variants[k].push((r.clone(), pair()?));
|
||||
}
|
||||
}
|
||||
replay.push(r);
|
||||
}
|
||||
let variant = |v: &[(Scored, Scored)]| VariantReading {
|
||||
n: v.len() as u32,
|
||||
chrf: mean(v.iter().map(|(_, x)| x.0.chrf)),
|
||||
meaning: mean(v.iter().filter_map(|(_, x)| x.0.meaning)),
|
||||
moved: v.iter().filter(|(r, x)| x.0.chrf < r.0.chrf).count() as u32,
|
||||
ref_gain_chrf: mean(v.iter().map(|(r, x)| r.1.chrf - x.1.chrf)),
|
||||
ref_gain_meaning: mean(
|
||||
v.iter().filter_map(|(r, x)| Some(r.1.meaning? - x.1.meaning?)),
|
||||
),
|
||||
};
|
||||
readings.push((
|
||||
*split,
|
||||
AblationReading {
|
||||
n: rows.len() as u32,
|
||||
unusable: unusable[at],
|
||||
replay: Agreement {
|
||||
chrf: mean(replay.iter().map(|r| r.0.chrf)),
|
||||
meaning: mean(replay.iter().filter_map(|r| r.0.meaning)),
|
||||
},
|
||||
drop: variant(&variants[0]),
|
||||
permute: variant(&variants[1]),
|
||||
},
|
||||
));
|
||||
}
|
||||
Ok(AblationOutcome::Read(readings))
|
||||
}
|
||||
}
|
||||
|
||||
/// The distinct rule codes an admission refusal broke: validator rules by
|
||||
/// code, a store refusal by its variant name. Nothing from the packet or the
|
||||
/// stores is kept; an I/O failure is `io`.
|
||||
|
|
@ -1008,7 +1313,7 @@ pub fn round<M: ModelPort, E: Embedder>(
|
|||
}
|
||||
|
||||
/// The model and embedder the run pinned are the ones answering.
|
||||
fn check_drift<M: ModelPort, E: Embedder>(
|
||||
fn check_drift<M: ModelPort + ?Sized, E: Embedder>(
|
||||
run: &RunStores,
|
||||
model: &M,
|
||||
scorer: &Scorer<E>,
|
||||
|
|
|
|||
|
|
@ -1633,3 +1633,90 @@ fn loop_open_requires_the_lowerer_reach_and_pins_it() {
|
|||
weksa::run::LowererReach::BriefedSubagent
|
||||
);
|
||||
}
|
||||
|
||||
/// Answers the Dao encode prompts with the Dao packets and every lowering with
|
||||
/// one sentence.
|
||||
struct DaoModel;
|
||||
|
||||
impl weksa::model::ModelPort for DaoModel {
|
||||
fn model(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
fn answer(&mut self, prompt: &weksa::context::Prompt) -> Result<String, weksa::model::PortError> {
|
||||
if prompt.text().starts_with(weksa::context::LOWER_INSTRUCTIONS) {
|
||||
return Ok(r#"{"text":"The highest excellence is like the water."}"#.into());
|
||||
}
|
||||
let n: usize = prompt.item_keys()[0].rsplit(':').next().unwrap().parse().unwrap();
|
||||
Ok(serde_json::to_string(&weksa::PacketDraft::of(&weksa::daodejing::packets()[n - 1])).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_ablate_waits_then_prints_one_line_per_split() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
open_dao_run(d.path());
|
||||
let url = fake_embedder();
|
||||
let root = d.path().join("canaryroot");
|
||||
|
||||
// Record round 1 the way `loop round` does, then put its lowering answers
|
||||
// where the relay would have.
|
||||
let mut run = weksa::run::RunStores::open(&root).unwrap();
|
||||
let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder));
|
||||
let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap()
|
||||
else {
|
||||
panic!("answered")
|
||||
};
|
||||
for call in &r.calls()[3..] {
|
||||
std::fs::write(
|
||||
root.join(format!("round-1/{}.answer", call.prompt)),
|
||||
r#"{"text":"The highest excellence is like the water."}"#,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let ablate = || weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]);
|
||||
let first = ablate();
|
||||
assert_eq!(first.status.code(), Some(3), "{}", out(&first));
|
||||
let text = out(&first);
|
||||
let digests: Vec<&str> = text
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("unanswered "))
|
||||
.filter(|d| d.len() == 16)
|
||||
.collect();
|
||||
// Three replays, three drops and the Validation permutes (2 items).
|
||||
assert_eq!(digests.len(), 3 + 3 + 2, "{text}");
|
||||
for digest in &digests {
|
||||
assert!(root.join(format!("ablation-1/{digest}.prompt")).is_file());
|
||||
std::fs::write(
|
||||
root.join(format!("ablation-1/{digest}.answer")),
|
||||
r#"{"text":"The highest excellence is like the water."}"#,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let second = ablate();
|
||||
assert!(second.status.success(), "{}", out(&second));
|
||||
let text = out(&second);
|
||||
assert_eq!(
|
||||
text,
|
||||
"train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n"
|
||||
);
|
||||
for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] {
|
||||
assert!(!text.contains(leaked), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_ablate_without_the_rounds_answers_says_so() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
open_dao_run(d.path());
|
||||
let url = fake_embedder();
|
||||
let root = d.path().join("canaryroot");
|
||||
let mut run = weksa::run::RunStores::open(&root).unwrap();
|
||||
let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder));
|
||||
weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap();
|
||||
let o = weksa(d.path(), &["loop", "ablate", "canaryroot", "--round", "1", "--ollama-url", &url]);
|
||||
assert!(!o.status.success());
|
||||
let text = out(&o);
|
||||
assert!(text.contains("holds no answer") && text.contains("head inventory"), "{text}");
|
||||
assert!(!text.contains("would not read"), "{text}");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -630,6 +630,7 @@ mod lowering {
|
|||
for needed in [
|
||||
"en-US",
|
||||
"all of its meaning and nothing it does not carry",
|
||||
"a grade is permille, 0 to 1000",
|
||||
"felt affect and restraint",
|
||||
"shapes how the utterance is said, not what it says",
|
||||
"forms on the entities line",
|
||||
|
|
@ -640,6 +641,61 @@ mod lowering {
|
|||
// The one call it makes is its item; the key is in no text.
|
||||
assert_eq!(prompt.item_keys(), [item.as_str()]);
|
||||
assert!(!prompt.text().contains(item.as_str()));
|
||||
// The view's speaker and group are private: the item's speaker entity
|
||||
// id and its group are in no text.
|
||||
let held = CorpusStore::open(&data("daodejing/corpus.cc"))
|
||||
.unwrap()
|
||||
.get(item.as_str())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!held.group.is_empty());
|
||||
for private in [held.speaker.as_str(), held.group.as_str()] {
|
||||
assert!(!prompt.text().contains(private), "{private}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_prompt_moves_with_the_stance_alone() {
|
||||
use weksa::packet::{FeatValue, FeatureValue, Grade, Stance};
|
||||
let (_, inv, entities, item) = dao(2);
|
||||
let store = InventoryStore::open(&data("daodejing/inventory.cc")).unwrap();
|
||||
let own = packets().remove(2).stance;
|
||||
let grade = |feature: &str, v: u16| FeatureValue {
|
||||
feature: feature.parse().unwrap(),
|
||||
value: FeatValue::Grade(Grade::new(v).unwrap()),
|
||||
};
|
||||
let angry = Stance {
|
||||
features: vec![grade("feature:9", 800), grade("feature:18", 900)],
|
||||
};
|
||||
let prompt_of = |stance: Stance| {
|
||||
let mut packet = packets().remove(2);
|
||||
packet.stance = stance.clone();
|
||||
let view = admitted(packet, &store, "daodejing/corpus.cc", &entities);
|
||||
let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap();
|
||||
(stance, prompt)
|
||||
};
|
||||
let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())];
|
||||
|
||||
// Three stances, three prompts.
|
||||
let digests: std::collections::BTreeSet<String> =
|
||||
three.iter().map(|(_, p)| p.digest()).collect();
|
||||
assert_eq!(digests.len(), 3);
|
||||
// Each packet line ends with exactly that stance's JSON, and the prompt
|
||||
// with the packet line taken out is the same text for all three.
|
||||
let without_packet_line = |text: &str| -> String {
|
||||
text.lines().filter(|l| !l.starts_with("packet: ")).collect::<Vec<_>>().join("\n")
|
||||
};
|
||||
let bare = without_packet_line(three[0].1.text());
|
||||
for (stance, prompt) in &three {
|
||||
let line = prompt.text().lines().find(|l| l.starts_with("packet: ")).unwrap();
|
||||
let json = serde_json::to_string(stance).unwrap();
|
||||
assert!(line.ends_with(&format!(r#","stance":{json}}}"#)), "{line}");
|
||||
assert_eq!(without_packet_line(prompt.text()), bare);
|
||||
}
|
||||
// The stance's feature ids are glossed in the one inventory listing.
|
||||
let text = three[1].1.text();
|
||||
assert!(text.contains("\nfeature:9: anger [grade; slots Stance]\n"));
|
||||
assert!(text.contains("\nfeature:18: restraint [grade; slots Stance]\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -2598,9 +2598,19 @@ fn source_tripwires() {
|
|||
let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count();
|
||||
assert!(!run.contains("pub fn new"));
|
||||
// The round does not call the template table: the model lowers.
|
||||
assert_eq!(count("lower(&"), 0, "run.rs calls the table");
|
||||
assert_eq!(
|
||||
code.iter()
|
||||
.filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower("))
|
||||
.count(),
|
||||
0,
|
||||
"run.rs calls the table"
|
||||
);
|
||||
assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table");
|
||||
assert_eq!(count("Prompt::lower("), 1, "one lowering call site");
|
||||
assert_eq!(
|
||||
count("Prompt::lower("),
|
||||
2,
|
||||
"two lowering call sites: the round's and the stance ablation's replay of it"
|
||||
);
|
||||
// Prompt::lower takes the view, the item's key, the inventory and the entity
|
||||
// port: no corpus item, train set, offer or store.
|
||||
let start = context.find("pub fn lower(").unwrap();
|
||||
|
|
@ -3560,3 +3570,542 @@ fn own_family_counts_offered_items_of_the_answerer_family() {
|
|||
assert_eq!(mixed_family_round(ModelFamily::Google), 0);
|
||||
assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0);
|
||||
}
|
||||
|
||||
// ---- stance ablation (stance-to-lowerer) ----
|
||||
|
||||
/// A lowerer by function: the text it answers for an item key and the prompt
|
||||
/// text, recording every prompt it was shown.
|
||||
struct Lowerer {
|
||||
f: fn(&str, &str) -> String,
|
||||
seen: Vec<(String, String, String)>,
|
||||
}
|
||||
|
||||
impl Lowerer {
|
||||
fn new(f: fn(&str, &str) -> String) -> Self {
|
||||
Self { f, seen: vec![] }
|
||||
}
|
||||
/// The digests asked, once each, in the order first asked.
|
||||
fn digests(&self) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (d, ..) in &self.seen {
|
||||
if !out.contains(d) {
|
||||
out.push(d.clone());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelPort for Lowerer {
|
||||
fn model(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
|
||||
let key = prompt.item_keys()[0].clone();
|
||||
self.seen.push((prompt.digest(), key.clone(), prompt.text().to_string()));
|
||||
Ok(serde_json::json!({ "text": (self.f)(&key, prompt.text()) }).to_string())
|
||||
}
|
||||
}
|
||||
|
||||
const WRITTEN: &str = "item:written:1";
|
||||
|
||||
fn english(key: &str) -> String {
|
||||
let mut table = table_english();
|
||||
table.insert(
|
||||
WRITTEN.into(),
|
||||
"Heaven endures through the long years without end.".into(),
|
||||
);
|
||||
table[key].clone()
|
||||
}
|
||||
|
||||
/// The packet line's stance, as the prompt prints it: the packet line is one
|
||||
/// JSON object ending in the stance.
|
||||
fn stance_of(text: &str) -> String {
|
||||
let line = text.lines().find(|l| l.starts_with("packet: ")).unwrap();
|
||||
line[line.find("\"stance\":").unwrap()..].to_string()
|
||||
}
|
||||
|
||||
/// Says the same thing whatever the stance.
|
||||
fn blind(key: &str, _text: &str) -> String {
|
||||
english(key)
|
||||
}
|
||||
|
||||
/// Appends the stance it was shown.
|
||||
fn reader(key: &str, text: &str) -> String {
|
||||
format!("{} {}", english(key), stance_of(text))
|
||||
}
|
||||
|
||||
/// Does better with the item's own stance: the real sentence for it, nonsense
|
||||
/// for any other, empty included.
|
||||
fn helped(key: &str, text: &str) -> String {
|
||||
let r800 = r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}}]}}"#;
|
||||
let own = match key {
|
||||
"item:daodejing:3" => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}},{"feature":"feature:16","value":{"Grade":300}}]}}"#,
|
||||
WRITTEN => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":500}}]}}"#,
|
||||
_ => r800,
|
||||
};
|
||||
if stance_of(text) == own { english(key) } else { "zzz qqq xxx".into() }
|
||||
}
|
||||
|
||||
fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance {
|
||||
weksa::packet::Stance {
|
||||
features: features
|
||||
.iter()
|
||||
.map(|(f, v)| weksa::packet::FeatureValue {
|
||||
feature: f.parse().unwrap(),
|
||||
value: weksa::packet::FeatValue::Grade(common::grade(*v)),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A Dao run with a third Validation item (`item:written:1`, key order after
|
||||
/// items 2 and 3) and round 1 recorded over it, every item encoded to its Dao
|
||||
/// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3
|
||||
/// restraint 800 and affection 300; the written item restraint 500.
|
||||
fn recorded_dao(edit: impl FnOnce(&mut HashMap<String, String>)) -> (Dao, RoundRecord) {
|
||||
let mut dao = dao_over(|corpus, entities| {
|
||||
let entities = weksa::EntityStore::open(entities).unwrap();
|
||||
let mut store = CorpusStore::open(corpus).unwrap();
|
||||
let mut written = store.get("item:daodejing:2").unwrap().unwrap();
|
||||
written.id = WRITTEN.into();
|
||||
written.text = "Heaven endures through the long years without end.".into();
|
||||
written.source = weksa::corpus::Source::OperatorWritten;
|
||||
store.add(written, &entities).unwrap();
|
||||
});
|
||||
let mut model = fake(&good_answers());
|
||||
let mut extra = draft_of(2);
|
||||
extra.referents.truncate(1);
|
||||
extra.predications.truncate(1);
|
||||
extra.stance = grade_stance(&[("feature:18", 500)]);
|
||||
model.answers.insert(WRITTEN.into(), json(&extra));
|
||||
edit(&mut model.answers);
|
||||
model.lowered.insert(WRITTEN.into(), english(WRITTEN));
|
||||
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
(dao, r)
|
||||
}
|
||||
|
||||
fn ablate(
|
||||
dao: &Dao,
|
||||
recorded: &mut dyn ModelPort,
|
||||
model: &mut dyn ModelPort,
|
||||
) -> Result<weksa::run::AblationOutcome, RunError> {
|
||||
dao.run.stance_ablation(1, recorded, model, &Scorer::new(Bag))
|
||||
}
|
||||
|
||||
fn read(
|
||||
outcome: weksa::run::AblationOutcome,
|
||||
) -> Vec<(weksa::Split, weksa::run::AblationReading)> {
|
||||
match outcome {
|
||||
weksa::run::AblationOutcome::Read(r) => r,
|
||||
weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_answers_read_and_never_write() {
|
||||
use std::io::ErrorKind;
|
||||
use weksa::model::RecordedAnswers;
|
||||
let p = prompt();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut port = RecordedAnswers::new(dir.path().to_path_buf(), "answerer".into());
|
||||
assert_eq!(port.model(), "answerer");
|
||||
// A miss is Unanswered and the directory is as it was.
|
||||
assert_eq!(port.answer(&p), Err(PortError::Unanswered));
|
||||
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
|
||||
// A directory that does not exist is a miss too, and is not made.
|
||||
let mut absent = RecordedAnswers::new(dir.path().join("nope"), "m".into());
|
||||
assert_eq!(absent.answer(&p), Err(PortError::Unanswered));
|
||||
assert!(!dir.path().join("nope").exists());
|
||||
// A hit is the file's text as written.
|
||||
let file = dir.path().join(format!("{}.answer", p.digest()));
|
||||
std::fs::write(&file, " {\"x\": 1}\n").unwrap();
|
||||
assert_eq!(port.answer(&p).unwrap(), " {\"x\": 1}\n");
|
||||
std::fs::write(&file, [0xff, 0xfe]).unwrap();
|
||||
assert_eq!(port.answer(&p), Err(PortError::Io(ErrorKind::InvalidData)));
|
||||
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stance_ablation_asks_replay_drop_and_permute_for_train_and_validation_only() {
|
||||
let (dao, r) = recorded_dao(|_| {});
|
||||
let round_digests: Vec<String> = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect();
|
||||
assert_eq!(round_digests.len(), 4, "the round lowered four items");
|
||||
let mut recorded_port = Lowerer::new(reader);
|
||||
let mut model = Lowerer::new(reader);
|
||||
read(ablate(&dao, &mut recorded_port, &mut model).unwrap());
|
||||
|
||||
// The base prompts are the round's own, byte for byte.
|
||||
let recorded_digests = recorded_port.digests();
|
||||
assert_eq!(recorded_digests, round_digests);
|
||||
let asked = model.digests();
|
||||
for d in &round_digests {
|
||||
assert!(asked.contains(d), "the replay asks the round's prompt {d}");
|
||||
}
|
||||
// Per item: replay and drop for each; permute for each Validation item, and
|
||||
// none for the lone Train item.
|
||||
let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count();
|
||||
assert_eq!(per_key("item:daodejing:1"), 2);
|
||||
for key in ["item:daodejing:2", "item:daodejing:3", WRITTEN] {
|
||||
assert_eq!(per_key(key), 3, "{key}");
|
||||
}
|
||||
// Test items are never asked.
|
||||
for (_, key, text) in &model.seen {
|
||||
assert!(key != "item:daodejing:4" && key != "item:daodejing:5", "{key}");
|
||||
assert!(!text.contains("item:daodejing:4"));
|
||||
}
|
||||
// A drop prompt prints the empty stance, a replay the item's own.
|
||||
let empty = model
|
||||
.seen
|
||||
.iter()
|
||||
.filter(|s| stance_of(&s.2) == "\"stance\":{\"features\":[]}}")
|
||||
.count();
|
||||
assert_eq!(empty, 4, "one drop per item");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let one = |f: fn(&str, &str) -> String| {
|
||||
let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f));
|
||||
read(ablate(&dao, &mut rec, &mut model).unwrap())
|
||||
};
|
||||
for (split, r) in one(blind) {
|
||||
assert_eq!(r.unusable, 0);
|
||||
assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line());
|
||||
assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9);
|
||||
assert!((r.drop.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9);
|
||||
if r.permute.n > 0 {
|
||||
assert!((r.permute.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9);
|
||||
}
|
||||
assert_eq!(r.drop.ref_gain_chrf, Some(0.0));
|
||||
}
|
||||
let reading = one(reader);
|
||||
let (train, validation) = (&reading[0].1, &reading[1].1);
|
||||
assert_eq!((reading[0].0, reading[1].0), (weksa::Split::Train, weksa::Split::Validation));
|
||||
assert_eq!((train.n, validation.n), (1, 3));
|
||||
assert_eq!((train.drop.n, train.drop.moved), (1, 1));
|
||||
assert_eq!((train.permute.n, train.permute.moved), (0, 0));
|
||||
assert_eq!((validation.drop.n, validation.drop.moved), (3, 3));
|
||||
assert_eq!((validation.permute.n, validation.permute.moved), (3, 3));
|
||||
// A lone Train item has no permute: no measurement, not total disagreement.
|
||||
assert_eq!(
|
||||
train.permute,
|
||||
weksa::run::VariantReading::default(),
|
||||
"{}",
|
||||
train.line()
|
||||
);
|
||||
assert!(
|
||||
train.line().contains(
|
||||
"| permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none"
|
||||
),
|
||||
"{}",
|
||||
train.line()
|
||||
);
|
||||
assert!((validation.replay.chrf.unwrap() - 100.0).abs() < 1e-9);
|
||||
assert!(validation.drop.chrf.unwrap() < validation.replay.chrf.unwrap());
|
||||
assert!(validation.permute.chrf.unwrap() < validation.replay.chrf.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let one = |f: fn(&str, &str) -> String| {
|
||||
let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f));
|
||||
read(ablate(&dao, &mut rec, &mut model).unwrap())
|
||||
};
|
||||
for (split, r) in one(helped) {
|
||||
assert!(r.drop.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line());
|
||||
assert!(r.drop.ref_gain_meaning.unwrap() > 0.0, "{split:?}: {}", r.line());
|
||||
if r.permute.n > 0 {
|
||||
assert!(r.permute.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line());
|
||||
}
|
||||
assert_eq!(r.drop.moved, r.drop.n);
|
||||
}
|
||||
for (split, r) in one(blind) {
|
||||
assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (Some(0.0), Some(0.0)), "{split:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_packet_without_stance_asks_no_drop_and_a_test_packet_in_the_round_is_never_asked() {
|
||||
let (dao, _) = recorded_dao(|answers| {
|
||||
let mut d = draft_of(3);
|
||||
d.stance = Default::default();
|
||||
answers.insert("item:daodejing:3".into(), json(&d));
|
||||
});
|
||||
// A packet for a Test item, put into the round's own store the way no round
|
||||
// would: the ablation reads splits from the corpus, not from the store.
|
||||
{
|
||||
let corpus = CorpusStore::open(&dao.corpus).unwrap();
|
||||
let inv = dao.run.inventory();
|
||||
let mut packet = packets().remove(3);
|
||||
packet.inventory_rev = 3;
|
||||
PacketStore::open(&dao.root.join("round-1/packets.cc"))
|
||||
.unwrap()
|
||||
.admit(packet, inv, &corpus, dao.run.entities())
|
||||
.unwrap();
|
||||
}
|
||||
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
|
||||
read(ablate(&dao, &mut rec, &mut model).unwrap());
|
||||
let bare = "\"stance\":{\"features\":[]}}";
|
||||
let drops_of_3 = model
|
||||
.seen
|
||||
.iter()
|
||||
.filter(|s| s.1 == "item:daodejing:3" && stance_of(&s.2) == bare)
|
||||
.count();
|
||||
// Its replay already shows the empty stance; there is no drop beside it.
|
||||
assert_eq!(drops_of_3, 1);
|
||||
assert!(model.seen.iter().all(|s| s.1 != "item:daodejing:4"), "a Test item is never asked");
|
||||
assert!(rec.seen.iter().all(|s| s.1 != "item:daodejing:4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn permute_takes_the_next_items_stance_in_its_split_and_skips_equal_ones() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
|
||||
read(ablate(&dao, &mut rec, &mut model).unwrap());
|
||||
// The stance each Validation item's permute prompt shows: the next item's
|
||||
// in key order (2, 3, written:1), cyclic; item 1 is alone in Train.
|
||||
let stance = |text: &str| stance_of(text);
|
||||
let of = |key: &str| -> Vec<String> {
|
||||
model
|
||||
.seen
|
||||
.iter()
|
||||
.filter(|s| s.1 == key)
|
||||
.map(|s| stance(&s.2))
|
||||
.collect()
|
||||
};
|
||||
let (s2, s3, sw) = (
|
||||
stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:2").unwrap().2),
|
||||
stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:3").unwrap().2),
|
||||
stance(&rec.seen.iter().find(|s| s.1 == WRITTEN).unwrap().2),
|
||||
);
|
||||
assert!(s2 != s3 && s3 != sw && sw != s2);
|
||||
assert!(of("item:daodejing:2").contains(&s3));
|
||||
assert!(of("item:daodejing:3").contains(&sw));
|
||||
assert!(of(WRITTEN).contains(&s2), "the last wraps to the first");
|
||||
assert!(!of("item:daodejing:2").contains(&sw), "not the previous or the other");
|
||||
// Item 1 is alone in Train: it never takes a stance across splits.
|
||||
assert_eq!(of("item:daodejing:1").len(), 2, "replay and drop only");
|
||||
|
||||
// Items 2 and 3 with equal stances: item 2's permute is its own prompt, so
|
||||
// it asks none; the others still do.
|
||||
let (dao, _) = recorded_dao(|answers| {
|
||||
let mut d = draft_of(3);
|
||||
d.stance = draft_of(2).stance;
|
||||
answers.insert("item:daodejing:3".into(), json(&d));
|
||||
});
|
||||
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
|
||||
let r = read(ablate(&dao, &mut rec, &mut model).unwrap());
|
||||
let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count();
|
||||
assert_eq!(per_key("item:daodejing:2"), 2, "replay and drop: its next holds the stance it has");
|
||||
assert_eq!(per_key("item:daodejing:3"), 3);
|
||||
assert_eq!(per_key(WRITTEN), 3);
|
||||
assert_eq!(r[1].1.permute.n, 2, "{}", r[1].1.line());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() {
|
||||
use weksa::model::RecordedAnswers;
|
||||
let (dao, r) = recorded_dao(|_| {});
|
||||
let round1 = dao.root.join("round-1");
|
||||
for c in &r.calls()[4..] {
|
||||
std::fs::write(
|
||||
round1.join(format!("{}.answer", c.prompt)),
|
||||
serde_json::json!({ "text": english(&c.items[0]) }).to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let snapshot = |dir: &Path| -> Vec<(String, Vec<u8>)> {
|
||||
let mut files: Vec<_> = std::fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.map(|e| {
|
||||
let e = e.unwrap();
|
||||
let bytes = if e.path().is_dir() { vec![] } else { std::fs::read(e.path()).unwrap() };
|
||||
(e.file_name().to_string_lossy().into_owned(), bytes)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
files.retain(|f| f.0 != "ablation-1");
|
||||
files
|
||||
};
|
||||
let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap());
|
||||
let go = || {
|
||||
let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into());
|
||||
let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into());
|
||||
// Nothing answers yet: the replay and variant prompts are written.
|
||||
dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag))
|
||||
};
|
||||
let weksa::run::AblationOutcome::Unanswered { prompts } = go().unwrap() else {
|
||||
panic!("nobody answered")
|
||||
};
|
||||
// Four replays (the round's own prompts), four drops, three permutes (item 1
|
||||
// is alone in Train).
|
||||
assert_eq!(prompts.len(), 4 + 4 + 3, "{prompts:?}");
|
||||
for d in &prompts {
|
||||
assert!(dao.run.ablation_dir(1).join(format!("{d}.prompt")).is_file());
|
||||
}
|
||||
// Answer every ask from a fake and read; then again over the same files.
|
||||
for d in &prompts {
|
||||
let text = std::fs::read_to_string(dao.run.ablation_dir(1).join(format!("{d}.prompt"))).unwrap();
|
||||
std::fs::write(
|
||||
dao.run.ablation_dir(1).join(format!("{d}.answer")),
|
||||
serde_json::json!({ "text": format!("answer {}", &text[..8]) }).to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let first = read(go().unwrap());
|
||||
let again = read(go().unwrap());
|
||||
assert_eq!(first, again);
|
||||
let after = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap());
|
||||
assert!(before.0 == after.0, "round-1 is unchanged byte for byte");
|
||||
assert!(before.1 == after.1, "the run root's files are unchanged");
|
||||
assert_eq!(before.2, after.2);
|
||||
assert_eq!(before.3, after.3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() {
|
||||
use weksa::model::RecordedAnswers;
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
// No round-2, and round 1 holds no answer files.
|
||||
let mut model = Lowerer::new(reader);
|
||||
let mut rec = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
|
||||
assert!(matches!(
|
||||
dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)),
|
||||
Err(RunError::Malformed("round"))
|
||||
));
|
||||
let Err(missing) = ablate(&dao, &mut rec, &mut model) else {
|
||||
panic!("a missing round answer is refused")
|
||||
};
|
||||
assert!(matches!(missing, RunError::Unrecorded));
|
||||
// It says what happened, not that a store would not read.
|
||||
let said = missing.to_string();
|
||||
assert!(said.contains("holds no answer") && said.contains("digest") && said.contains("head inventory"), "{said}");
|
||||
assert!(!said.contains("would not read"), "{said}");
|
||||
assert!(
|
||||
std::fs::read_dir(dao.root.join("round-1"))
|
||||
.unwrap()
|
||||
.all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".prompt")),
|
||||
"no prompt is written into the round"
|
||||
);
|
||||
// Drift is refused as a round refuses it.
|
||||
struct Named(Lowerer);
|
||||
impl ModelPort for Named {
|
||||
fn model(&self) -> &str {
|
||||
"other-model"
|
||||
}
|
||||
fn answer(&mut self, p: &Prompt) -> Result<String, PortError> {
|
||||
self.0.answer(p)
|
||||
}
|
||||
}
|
||||
let mut named = Named(Lowerer::new(reader));
|
||||
let mut rec2 = Lowerer::new(reader);
|
||||
assert!(matches!(
|
||||
ablate(&dao, &mut rec2, &mut named),
|
||||
Err(RunError::Drift(DriftOf::Model))
|
||||
));
|
||||
assert!(matches!(
|
||||
dao.run.stance_ablation(1, &mut rec2, &mut Lowerer::new(reader), &Scorer::new(OtherEmbedder)),
|
||||
Err(RunError::Drift(DriftOf::Embedder))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stance_ablation_refuses_a_cycle_round() {
|
||||
let mut dao = dao();
|
||||
two_rounds(&mut dao);
|
||||
run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT));
|
||||
let rounds = dao.run.rounds().unwrap();
|
||||
let candidate = rounds.iter().find(|r| r.cycle().is_some()).expect("the cycle's round");
|
||||
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
|
||||
assert!(matches!(
|
||||
dao.run.stance_ablation(candidate.n(), &mut rec, &mut model, &Scorer::new(Bag)),
|
||||
Err(RunError::Malformed("round"))
|
||||
));
|
||||
assert!(model.seen.is_empty() && rec.seen.is_empty());
|
||||
}
|
||||
|
||||
/// The real sentence for the item's own stance, an empty text (not a usable
|
||||
/// lowering) for any other.
|
||||
fn mute_unless_own(key: &str, text: &str) -> String {
|
||||
let said = helped(key, text);
|
||||
if said == "zzz qqq xxx" { String::new() } else { said }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unusable_variant_answers_are_counted_and_read_as_none_not_zero() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (mut rec, mut model) = (Lowerer::new(mute_unless_own), Lowerer::new(mute_unless_own));
|
||||
let reading = read(ablate(&dao, &mut rec, &mut model).unwrap());
|
||||
let (train, validation) = (&reading[0].1, &reading[1].1);
|
||||
// Every variant answer is unusable: Train has one drop, Validation three
|
||||
// drops and three permutes. The items themselves (replay) still read.
|
||||
assert_eq!((train.n, train.unusable), (1, 1));
|
||||
assert_eq!((validation.n, validation.unusable), (3, 6));
|
||||
for r in [train, validation] {
|
||||
assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9);
|
||||
assert_eq!(r.drop.n, 0);
|
||||
assert_eq!(r.permute.n, 0);
|
||||
assert_eq!(r.drop, weksa::run::VariantReading::default());
|
||||
assert_eq!(r.permute, weksa::run::VariantReading::default());
|
||||
}
|
||||
assert_eq!(
|
||||
train.line(),
|
||||
"ablation n=1 unusable=1 replay chrf=100.00 meaning=1.000 | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_line_names_each_variant_with_its_own_n_and_the_splits_n() {
|
||||
use weksa::run::{AblationReading, Agreement, VariantReading};
|
||||
let reading = AblationReading {
|
||||
n: 7,
|
||||
unusable: 2,
|
||||
replay: Agreement { chrf: Some(91.5), meaning: Some(0.9) },
|
||||
drop: VariantReading {
|
||||
n: 5,
|
||||
chrf: Some(80.25),
|
||||
meaning: Some(0.8),
|
||||
moved: 4,
|
||||
ref_gain_chrf: Some(3.5),
|
||||
ref_gain_meaning: Some(-0.125),
|
||||
},
|
||||
permute: VariantReading {
|
||||
n: 3,
|
||||
chrf: Some(70.0),
|
||||
meaning: None,
|
||||
moved: 1,
|
||||
ref_gain_chrf: None,
|
||||
ref_gain_meaning: Some(0.5),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
reading.line(),
|
||||
"ablation n=7 unusable=2 replay chrf=91.50 meaning=0.900 | drop n=5 chrf=80.25 meaning=0.800 moved=4 ref_gain_chrf=3.50 ref_gain_meaning=-0.125 | permute n=3 chrf=70.00 meaning=none moved=1 ref_gain_chrf=none ref_gain_meaning=0.500"
|
||||
);
|
||||
}
|
||||
|
||||
fn nothing_usable(_: &str, _: &str) -> String {
|
||||
String::new()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_split_where_no_replay_is_usable_reads_none_throughout() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(nothing_usable));
|
||||
let reading = read(ablate(&dao, &mut rec, &mut model).unwrap());
|
||||
// Train: the item and its drop; Validation: three items, three drops, three permutes.
|
||||
assert_eq!((reading[0].1.n, reading[0].1.unusable), (0, 2));
|
||||
assert_eq!((reading[1].1.n, reading[1].1.unusable), (0, 9));
|
||||
for (_, r) in &reading {
|
||||
assert_eq!(r.replay, weksa::run::Agreement::default());
|
||||
assert_eq!(r.drop, weksa::run::VariantReading::default());
|
||||
assert_eq!(r.permute, weksa::run::VariantReading::default());
|
||||
}
|
||||
assert_eq!(
|
||||
reading[0].1.line(),
|
||||
"ablation n=0 unusable=2 replay chrf=none meaning=none | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue