weksa loop relower: fixed-packet re-lowering measure with a lowering-only floor #69
5 changed files with 932 additions and 54 deletions
|
|
@ -13,9 +13,9 @@ use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
|
|||
use weksa::inventory::{proof_revision, seed, speaker_scene_revision};
|
||||
use weksa::packet::{FeatValue, FeatureValue, Grade};
|
||||
use weksa::packet::proof_packet;
|
||||
use weksa::model::RecordedAnswers;
|
||||
use weksa::model::{ModelPort, RecordedAnswers};
|
||||
use weksa::run::{
|
||||
AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
|
||||
AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
|
||||
};
|
||||
use weksa::score::ScoreInput;
|
||||
use weksa::{
|
||||
|
|
@ -153,6 +153,26 @@ enum LoopCommand {
|
|||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
/// Re-lower round ROUND's Validation packets through arm ARM, score them
|
||||
/// against the items, and, with --floor ARM, decide against that second arm
|
||||
/// by the panel accept rule. Prompts and answers live in
|
||||
/// `<ROOT>/relower-<ROUND>-<ARM>`; the command exits 3 until the arm is
|
||||
/// answered. --lowerer names the arm's model in the printed line. A
|
||||
/// diagnostic, never a gate; records nothing; Test and Train are never
|
||||
/// touched. Labels match [a-z0-9-]{1,32} and differ.
|
||||
Relower {
|
||||
root: PathBuf,
|
||||
#[arg(long)]
|
||||
round: u32,
|
||||
#[arg(long)]
|
||||
arm: String,
|
||||
#[arg(long)]
|
||||
lowerer: String,
|
||||
#[arg(long)]
|
||||
floor: Option<String>,
|
||||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
/// Run one edit-accept cycle of the run at ROOT. The stores are read only
|
||||
/// from the run's record. The editor's prompt, then the candidate round's
|
||||
/// encode prompts, then its lowering prompts are written in turn; the
|
||||
|
|
@ -1115,6 +1135,59 @@ fn main() -> Result<()> {
|
|||
}
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Relower {
|
||||
root,
|
||||
round,
|
||||
arm,
|
||||
lowerer,
|
||||
floor,
|
||||
ollama_url,
|
||||
}) => {
|
||||
let label = |l: &str| {
|
||||
(1..=32).contains(&l.len())
|
||||
&& l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
|
||||
};
|
||||
if !label(&arm) || floor.as_deref().is_some_and(|f| !label(f) || f == arm) {
|
||||
anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ");
|
||||
}
|
||||
let run = RunStores::open(&root)?;
|
||||
let mut recorded = RecordedAnswers::new(
|
||||
root.join(format!("round-{round}")),
|
||||
run.record().model.clone(),
|
||||
);
|
||||
let mut files = run.answer_files(run.relower_dir(round, &arm))?;
|
||||
let mut floor_port = floor.as_deref().map(|f| {
|
||||
RecordedAnswers::new(run.relower_dir(round, f), run.record().model.clone())
|
||||
});
|
||||
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
|
||||
match run.relower(
|
||||
round,
|
||||
&arm,
|
||||
&mut recorded,
|
||||
&mut files,
|
||||
floor_port.as_mut().map(|p| p as &mut dyn ModelPort),
|
||||
&scorer,
|
||||
)? {
|
||||
RelowerOutcome::Unanswered { prompts } => {
|
||||
for digest in &prompts {
|
||||
println!("unanswered {digest}");
|
||||
}
|
||||
println!("unanswered {} prompts", prompts.len());
|
||||
std::process::exit(3);
|
||||
}
|
||||
RelowerOutcome::Read { languages, verdict } => {
|
||||
for l in languages {
|
||||
println!(
|
||||
"{}",
|
||||
run.labelled(&format!("{} arm={arm} lowerer={lowerer}", l.line()))
|
||||
);
|
||||
}
|
||||
if let Some(v) = verdict {
|
||||
println!("verdict {v:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Recall { root }) => {
|
||||
let run = RunStores::open(&root)?;
|
||||
let mut files = run.answer_files(run.recall_dir())?;
|
||||
|
|
|
|||
|
|
@ -495,6 +495,9 @@ pub enum RunError {
|
|||
Ontology,
|
||||
/// A scene does not read against the run's inventory.
|
||||
SpeakerState,
|
||||
/// The floor arm of a relower holds no answer to a prompt; it is a
|
||||
/// reference, never written to, and must be answered whole first.
|
||||
FloorUnanswered,
|
||||
}
|
||||
|
||||
impl fmt::Display for RunError {
|
||||
|
|
@ -523,6 +526,9 @@ impl fmt::Display for RunError {
|
|||
RunError::SpeakerState => {
|
||||
f.write_str("a scene does not read against the run's inventory")
|
||||
}
|
||||
RunError::FloorUnanswered => {
|
||||
f.write_str("the floor arm holds no answer to a lowering prompt; answer it whole first")
|
||||
}
|
||||
RunError::Panel => f.write_str(
|
||||
"the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item",
|
||||
),
|
||||
|
|
@ -1337,6 +1343,66 @@ type HeldSplit = (String, Split, Vec<(CorpusItem, Packet)>);
|
|||
type Lowered = (String, String, String, [Option<String>; 2]);
|
||||
|
||||
impl RunStores {
|
||||
/// The live items of each panel language in each listed split with their
|
||||
/// round-`n` packets, per language then split, sorted by item key. A round
|
||||
/// that is absent or a cycle's is `Malformed("round")`. Test is never held.
|
||||
fn held_splits(&self, n: u32, splits: &[Split]) -> Result<Vec<HeldSplit>, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
self.rounds()
|
||||
.map_err(store)?
|
||||
.into_iter()
|
||||
.find(|r| r.n() == n && r.cycle().is_none())
|
||||
.ok_or(RunError::Malformed("round"))?;
|
||||
let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc"))
|
||||
.map_err(store)?
|
||||
.all()
|
||||
.map_err(store)?;
|
||||
let mut held: Vec<HeldSplit> = Vec::new();
|
||||
for tag in &self.record.panel {
|
||||
for split in splits {
|
||||
held.push((tag.clone(), *split, Vec::new()));
|
||||
}
|
||||
}
|
||||
for packet in round_packets {
|
||||
let item = self
|
||||
.corpus
|
||||
.get(packet.provenance.corpus_item.as_str())
|
||||
.map_err(store)?
|
||||
.ok_or(RunError::Store)?;
|
||||
for (language, split, items) in &mut held {
|
||||
if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) {
|
||||
items.push((item.clone(), packet.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (_, _, items) in &mut held {
|
||||
items.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str()));
|
||||
}
|
||||
Ok(held)
|
||||
}
|
||||
|
||||
/// The lowering prompt of `packet` for `item`: admitted into `store`,
|
||||
/// assembled and lowered. The one builder for the round's prompt, which
|
||||
/// ablation and relower must reproduce byte for byte. `None` when the
|
||||
/// lowering prompt itself is refused, as it was for the round.
|
||||
fn lowering_prompt(
|
||||
&self,
|
||||
store: &mut PacketStore,
|
||||
packet: Packet,
|
||||
item: &CorpusItem,
|
||||
inventory: &crate::Inventory,
|
||||
ontology: &crate::Ontology,
|
||||
) -> Result<Option<Prompt>, RunError> {
|
||||
let packet_item = packet.provenance.corpus_item.clone();
|
||||
let view = store
|
||||
.admit(packet, &self.inventory, &self.corpus, &self.entities)
|
||||
.map_err(|_| RunError::Store)?;
|
||||
let view = self
|
||||
.assembled(view, &self.inventory)
|
||||
.map_err(|_| RunError::Store)?;
|
||||
Ok(Prompt::lower(&view, &packet_item, &item.language, inventory, &self.entities, ontology).ok())
|
||||
}
|
||||
|
||||
/// `<root>/ablation-<n>`, where a stance ablation's prompts, answers and
|
||||
/// variant packet stores live.
|
||||
pub fn ablation_dir(&self, n: u32) -> PathBuf {
|
||||
|
|
@ -1362,11 +1428,8 @@ impl RunStores {
|
|||
) -> Result<AblationOutcome, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
check_drift(self, model, scorer)?;
|
||||
self.rounds()
|
||||
.map_err(store)?
|
||||
.into_iter()
|
||||
.find(|r| r.n() == n && r.cycle().is_none())
|
||||
.ok_or(RunError::Malformed("round"))?;
|
||||
// A refused round writes nothing: the round is read before any folder is made.
|
||||
let mut splits = self.held_splits(n, &[Split::Train, Split::Validation])?;
|
||||
let inventory = self.inventory.inventory().map_err(store)?;
|
||||
let ontology = self.ontology.ontology().map_err(store)?;
|
||||
|
||||
|
|
@ -1383,56 +1446,17 @@ impl RunStores {
|
|||
PacketStore::open(&dir.join("permute.cc")).map_err(store)?,
|
||||
];
|
||||
|
||||
// Live Train and Validation packets, in item key order within a split.
|
||||
let round_packets = PacketStore::open(&self.root.join(format!("round-{n}")).join("packets.cc"))
|
||||
.map_err(store)?
|
||||
.all()
|
||||
.map_err(store)?;
|
||||
let mut splits: Vec<HeldSplit> = Vec::new();
|
||||
for tag in &self.record.panel {
|
||||
for split in [Split::Train, Split::Validation] {
|
||||
splits.push((tag.clone(), split, Vec::new()));
|
||||
}
|
||||
}
|
||||
for packet in round_packets {
|
||||
let item = self
|
||||
.corpus
|
||||
.get(packet.provenance.corpus_item.as_str())
|
||||
.map_err(store)?
|
||||
.ok_or(RunError::Store)?;
|
||||
for (language, split, held) in &mut splits {
|
||||
if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) {
|
||||
held.push((item.clone(), packet.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut waiting: Vec<String> = Vec::new();
|
||||
let mut readings = Vec::new();
|
||||
// Texts per split: (item text, recorded, replay, [drop, permute]).
|
||||
let mut texts: Vec<Vec<Lowered>> = Vec::new();
|
||||
let mut unusable = vec![0u32; splits.len()];
|
||||
for (at, (_, _, held)) in splits.iter_mut().enumerate() {
|
||||
held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str()));
|
||||
let mut rows = Vec::new();
|
||||
for (i, (item, packet)) in held.iter().enumerate() {
|
||||
let packet_item = packet.provenance.corpus_item.clone();
|
||||
let mut prompt_of = |slot: usize, packet: Packet| -> Result<Prompt, RunError> {
|
||||
let view = stores[slot]
|
||||
.admit(packet, &self.inventory, &self.corpus, &self.entities)
|
||||
.map_err(|_| RunError::Store)?;
|
||||
let view = self
|
||||
.assembled(view, &self.inventory)
|
||||
.map_err(|_| RunError::Store)?;
|
||||
Prompt::lower(
|
||||
&view,
|
||||
&packet_item,
|
||||
&item.language,
|
||||
&inventory,
|
||||
&self.entities,
|
||||
&ontology,
|
||||
)
|
||||
.map_err(|_| RunError::Store)
|
||||
self.lowering_prompt(&mut stores[slot], packet, item, &inventory, &ontology)?
|
||||
.ok_or(RunError::Store)
|
||||
};
|
||||
let base = prompt_of(0, packet.clone())?;
|
||||
let mut variants: [Option<Prompt>; 2] = [None, None];
|
||||
|
|
@ -1547,6 +1571,236 @@ impl RunStores {
|
|||
}
|
||||
}
|
||||
|
||||
/// One panel language's relower reading: Validation items only.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct RelowerReading {
|
||||
pub language: String,
|
||||
/// Live Validation items of the language read: those with a round packet
|
||||
/// whose lowering prompt the round could build. Every reading below is over
|
||||
/// these items, so recorded, arm and floor stay comparable.
|
||||
pub n: u32,
|
||||
/// Live Validation items of the language not read: refused by the round
|
||||
/// before admission (no packet) or at the lowering prompt. They are in no
|
||||
/// reading, so `recorded` can differ from the round's own Validation
|
||||
/// reading, which scores them 0.
|
||||
pub skipped: u32,
|
||||
/// Arm answers that were not a usable lowering; each scored 0.
|
||||
pub unusable: u32,
|
||||
/// Floor-arm answers that were not a usable lowering; each scored 0, which
|
||||
/// widens `floor`. Zero without a floor arm.
|
||||
pub floor_unusable: u32,
|
||||
/// The round's own lowering, read against the items.
|
||||
pub recorded: Reading,
|
||||
pub arm: Reading,
|
||||
/// `spread(recorded, floor arm)`, when a floor arm was given.
|
||||
pub floor: Option<Reading>,
|
||||
}
|
||||
|
||||
impl RelowerReading {
|
||||
pub fn line(&self) -> String {
|
||||
let show = |r: &Reading| {
|
||||
format!("chrf={:.2} meaning={:.3} composite={:.4}", r.chrf, r.meaning, r.composite)
|
||||
};
|
||||
let floor = self
|
||||
.floor
|
||||
.as_ref()
|
||||
.map_or(String::new(), |f| {
|
||||
format!(" | floor {} unusable={}", show(f), self.floor_unusable)
|
||||
});
|
||||
format!(
|
||||
"relower {} n={} skipped={} unusable={} recorded {} | arm {}{floor}",
|
||||
self.language,
|
||||
self.n,
|
||||
self.skipped,
|
||||
self.unusable,
|
||||
show(&self.recorded),
|
||||
show(&self.arm),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum RelowerOutcome {
|
||||
Unanswered { prompts: Vec<String> },
|
||||
/// Per panel language in panel order; the verdict only when a floor arm was given.
|
||||
Read { languages: Vec<RelowerReading>, verdict: Option<Verdict> },
|
||||
}
|
||||
|
||||
impl RunStores {
|
||||
/// `<root>/relower-<n>-<arm>`, where an arm's prompts and answers live.
|
||||
pub fn relower_dir(&self, n: u32, arm: &str) -> PathBuf {
|
||||
self.root.join(format!("relower-{n}-{arm}"))
|
||||
}
|
||||
|
||||
/// A diagnostic like [`recall`](Self::recall) and
|
||||
/// [`stance_ablation`](Self::stance_ablation), never a gate: it serves the
|
||||
/// distil decision, whether a tuned lowerer is worth keeping against the
|
||||
/// prompted one (calibration-pending: see the map). Asks `arm` the byte-identical lowering prompts of round `n`'s live
|
||||
/// Validation items, reads the arm and the round's own lowering
|
||||
/// (`recorded`) against the items, and, given a `floor` arm (a second
|
||||
/// prompted lowering, so the floor is lowering-only noise), decides by
|
||||
/// [`decide_panel`] over (recorded, arm, spread(recorded, floor)). Only
|
||||
/// the model of `arm` is unpinned; the embedder is checked. An arm answer
|
||||
/// that is not a usable lowering scores 0 and is counted in `unusable`.
|
||||
/// Records nothing; `arm_label` names only where the scratch packet store
|
||||
/// lives, in the arm's directory.
|
||||
pub fn relower<E: Embedder>(
|
||||
&self,
|
||||
n: u32,
|
||||
arm_label: &str,
|
||||
recorded: &mut dyn ModelPort,
|
||||
arm: &mut dyn ModelPort,
|
||||
mut floor: Option<&mut dyn ModelPort>,
|
||||
scorer: &Scorer<E>,
|
||||
) -> Result<RelowerOutcome, RunError> {
|
||||
let store = |_| RunError::Store;
|
||||
check_embedder(self, scorer)?;
|
||||
let held = self.held_splits(n, &[Split::Validation])?;
|
||||
let inventory = self.inventory.inventory().map_err(store)?;
|
||||
let ontology = self.ontology.ontology().map_err(store)?;
|
||||
|
||||
let dir = self.relower_dir(n, arm_label);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?;
|
||||
for suffix in ["cc", "cc.lock"] {
|
||||
let _ = std::fs::remove_file(dir.join(format!("base.{suffix}")));
|
||||
}
|
||||
let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?;
|
||||
|
||||
// Per language, per item: the item text, then the recorded, arm and floor
|
||||
// lowerings (`None` when unusable).
|
||||
type Row = (String, Option<String>, Option<String>, Option<String>);
|
||||
let mut waiting: Vec<String> = Vec::new();
|
||||
let mut floor_waiting = false;
|
||||
let mut rows: Vec<Vec<Row>> = Vec::new();
|
||||
for (_, _, items) in &held {
|
||||
let mut language_rows = Vec::new();
|
||||
for (item, packet) in items {
|
||||
// An item the round could not lower has no prompt to ask; it is skipped
|
||||
// and counted, not scored.
|
||||
let Some(prompt) =
|
||||
self.lowering_prompt(&mut scratch, packet.clone(), item, &inventory, &ontology)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let text = |a: Asked| match a {
|
||||
Asked::Text(t) => Some(t),
|
||||
_ => None,
|
||||
};
|
||||
let recorded_text = match asked(recorded, &prompt)? {
|
||||
Asked::Waiting => return Err(RunError::Unrecorded),
|
||||
other => text(other),
|
||||
};
|
||||
let floor_text = match floor.as_deref_mut() {
|
||||
Some(f) => match asked(f, &prompt)? {
|
||||
Asked::Waiting => {
|
||||
floor_waiting = true;
|
||||
None
|
||||
}
|
||||
other => text(other),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let arm_text = match asked(arm, &prompt)? {
|
||||
Asked::Waiting => {
|
||||
let d = prompt.digest();
|
||||
if !waiting.contains(&d) {
|
||||
waiting.push(d);
|
||||
}
|
||||
None
|
||||
}
|
||||
other => text(other),
|
||||
};
|
||||
language_rows.push((item.text.clone(), recorded_text, arm_text, floor_text));
|
||||
}
|
||||
rows.push(language_rows);
|
||||
}
|
||||
if floor_waiting {
|
||||
return Err(RunError::FloorUnanswered);
|
||||
}
|
||||
if !waiting.is_empty() {
|
||||
return Ok(RelowerOutcome::Unanswered { prompts: waiting });
|
||||
}
|
||||
|
||||
// One score call: each usable lowering against its item, in row order.
|
||||
let mut inputs: Vec<ScoreInput> = Vec::new();
|
||||
for (reference, recorded, arm, floor_text) in rows.iter().flatten() {
|
||||
for output in [recorded, arm, floor_text].into_iter().flatten() {
|
||||
inputs.push(ScoreInput { reference, output });
|
||||
}
|
||||
}
|
||||
let mut scores = if inputs.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
scorer.score(&inputs).map_err(|_| RunError::Embed)?
|
||||
}
|
||||
.into_iter();
|
||||
let mut next = |t: &Option<String>| -> Result<ItemScore, RunError> {
|
||||
match t {
|
||||
Some(_) => scores.next().ok_or(RunError::Store),
|
||||
None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }),
|
||||
}
|
||||
};
|
||||
let reading_of = |scored: &[ItemScore]| {
|
||||
Reading::of(&SplitMean {
|
||||
count: scored.len(),
|
||||
chrf: mean(scored.iter().map(|s| s.chrf)),
|
||||
meaning: mean(scored.iter().filter_map(|s| s.meaning)),
|
||||
})
|
||||
.ok_or(RunError::NoValidation)
|
||||
};
|
||||
let mut languages = Vec::new();
|
||||
let live = self.corpus.all().map_err(store)?;
|
||||
for ((language, _, _), language_rows) in held.iter().zip(&rows) {
|
||||
let (mut rec, mut arm_scores, mut floor_scores) = (Vec::new(), Vec::new(), Vec::new());
|
||||
let (mut unusable, mut floor_unusable) = (0u32, 0u32);
|
||||
for (_, r, a, f) in language_rows {
|
||||
rec.push(next(r)?);
|
||||
arm_scores.push(next(a)?);
|
||||
if a.is_none() {
|
||||
unusable += 1;
|
||||
}
|
||||
if floor.is_some() {
|
||||
floor_scores.push(next(f)?);
|
||||
if f.is_none() {
|
||||
floor_unusable += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let recorded_reading = reading_of(&rec)?;
|
||||
languages.push(RelowerReading {
|
||||
language: language.clone(),
|
||||
n: language_rows.len() as u32,
|
||||
skipped: live
|
||||
.iter()
|
||||
.filter(|i| {
|
||||
&i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation)
|
||||
})
|
||||
.count() as u32
|
||||
- language_rows.len() as u32,
|
||||
unusable,
|
||||
floor_unusable,
|
||||
recorded: recorded_reading,
|
||||
arm: reading_of(&arm_scores)?,
|
||||
floor: if floor.is_some() {
|
||||
Some(spread(&recorded_reading, &reading_of(&floor_scores)?))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
});
|
||||
}
|
||||
let verdict = if floor.is_some() {
|
||||
let panel: Vec<(Reading, Reading, Reading)> = languages
|
||||
.iter()
|
||||
.filter_map(|l| Some((l.recorded, l.arm, l.floor?)))
|
||||
.collect();
|
||||
Some(decide_panel(&panel))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(RelowerOutcome::Read { languages, verdict })
|
||||
}
|
||||
}
|
||||
|
||||
/// The distinct rule codes an admission refusal broke: validator rules by
|
||||
/// code, a store refusal by its variant name. Nothing from the packet or the
|
||||
/// stores is kept; an I/O failure is `io`.
|
||||
|
|
@ -1613,6 +1867,11 @@ fn check_drift<M: ModelPort + ?Sized, E: Embedder>(
|
|||
if model.model() != run.record.model {
|
||||
return Err(RunError::Drift(DriftOf::Model));
|
||||
}
|
||||
check_embedder(run, scorer)
|
||||
}
|
||||
|
||||
/// The embedder the run pinned is the one scoring.
|
||||
fn check_embedder<E: Embedder>(run: &RunStores, scorer: &Scorer<E>) -> Result<(), RunError> {
|
||||
if scorer.embedder().model() != run.record.embedder {
|
||||
return Err(RunError::Drift(DriftOf::Embedder));
|
||||
}
|
||||
|
|
@ -2223,16 +2482,21 @@ fn floor_of(
|
|||
let (a, b) = (replicate(1, tag)?, replicate(2, tag)?);
|
||||
Ok((
|
||||
tag.clone(),
|
||||
Reading {
|
||||
chrf: (a.chrf - b.chrf).abs(),
|
||||
meaning: (a.meaning - b.meaning).abs(),
|
||||
composite: (a.composite - b.composite).abs(),
|
||||
},
|
||||
spread(&a, &b),
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The absolute difference of two readings, metric by metric.
|
||||
fn spread(a: &Reading, b: &Reading) -> Reading {
|
||||
Reading {
|
||||
chrf: (a.chrf - b.chrf).abs(),
|
||||
meaning: (a.meaning - b.meaning).abs(),
|
||||
composite: (a.composite - b.composite).abs(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Each panel language's Validation reading in `round`, in panel order.
|
||||
fn readings_of(
|
||||
round: &RoundRecord,
|
||||
|
|
|
|||
|
|
@ -1778,6 +1778,86 @@ fn loop_ablate_waits_then_prints_one_line_per_split() {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
open_dao_run(d.path());
|
||||
let url = fake_embedder();
|
||||
let root = d.path().join("canaryroot");
|
||||
let mut run = weksa::run::RunStores::open(&root).unwrap();
|
||||
let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder));
|
||||
let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap()
|
||||
else {
|
||||
panic!("answered")
|
||||
};
|
||||
let said = r#"{"text":"The highest excellence is like the water."}"#;
|
||||
for call in &r.calls()[3..] {
|
||||
std::fs::write(root.join(format!("round-1/{}.answer", call.prompt)), said).unwrap();
|
||||
}
|
||||
let relower = |extra: &[&str]| {
|
||||
weksa(
|
||||
d.path(),
|
||||
&[&["loop", "relower", "canaryroot", "--round", "1", "--lowerer", "tuned-x", "--ollama-url", &url], extra].concat(),
|
||||
)
|
||||
};
|
||||
|
||||
// Unanswered: exit 3, the digests, a prompt file per digest in the arm's directory.
|
||||
let first = relower(&["--arm", "tuned"]);
|
||||
assert_eq!(first.status.code(), Some(3), "{}", out(&first));
|
||||
let text = out(&first);
|
||||
let digests: Vec<&str> = text
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("unanswered "))
|
||||
.filter(|x| x.len() == 16)
|
||||
.collect();
|
||||
assert_eq!(digests.len(), 2, "the two Validation prompts: {text}");
|
||||
for digest in &digests {
|
||||
assert!(root.join(format!("relower-1-tuned/{digest}.prompt")).is_file());
|
||||
std::fs::write(root.join(format!("relower-1-tuned/{digest}.answer")), said).unwrap();
|
||||
}
|
||||
|
||||
// Answered: one line per language, labelled, naming the lowerer; no verdict without a floor.
|
||||
let second = relower(&["--arm", "tuned"]);
|
||||
assert!(second.status.success(), "{}", out(&second));
|
||||
let text = out(&second);
|
||||
assert!(text.starts_with("relower en-US n=2 skipped=0 unusable=0 recorded "), "{text}");
|
||||
assert!(text.contains("arm=tuned lowerer=tuned-x"), "{text}");
|
||||
assert!(text.contains(" lowering="), "{text}");
|
||||
assert!(!text.contains("verdict") && !text.contains("floor"), "{text}");
|
||||
for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] {
|
||||
assert!(!text.contains(leaked), "{text}");
|
||||
}
|
||||
|
||||
// A floor arm that is not answered whole is refused, and never written to.
|
||||
let o = relower(&["--arm", "tuned", "--floor", "second"]);
|
||||
assert_eq!(o.status.code(), Some(1), "{}", out(&o));
|
||||
assert!(!root.join("relower-1-second").exists());
|
||||
|
||||
// Answered floor: a verdict line.
|
||||
std::fs::create_dir(root.join("relower-1-second")).unwrap();
|
||||
for digest in &digests {
|
||||
std::fs::write(root.join(format!("relower-1-second/{digest}.answer")), said).unwrap();
|
||||
}
|
||||
let o = relower(&["--arm", "tuned", "--floor", "second"]);
|
||||
assert!(o.status.success(), "{}", out(&o));
|
||||
let text = out(&o);
|
||||
assert!(text.contains("| floor chrf=0.00") && text.trim_end().ends_with("verdict Rejected(NoGain)"), "{text}");
|
||||
|
||||
// Labels: equal, uppercase, empty, too long, a path.
|
||||
let long = "a".repeat(33);
|
||||
for args in [
|
||||
vec!["--arm", "same", "--floor", "same"],
|
||||
vec!["--arm", "Upper"],
|
||||
vec!["--arm", ""],
|
||||
vec!["--arm", &long],
|
||||
vec!["--arm", "../x"],
|
||||
vec!["--arm", "tuned", "--floor", "Bad"],
|
||||
] {
|
||||
let o = relower(&args);
|
||||
assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_ablate_without_the_rounds_answers_says_so() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
|
|
|
|||
|
|
@ -3764,9 +3764,18 @@ fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance {
|
|||
/// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3
|
||||
/// restraint 800 and affection 300; the written item restraint 500.
|
||||
fn recorded_dao(edit: impl FnOnce(&mut HashMap<String, String>)) -> (Dao, RoundRecord) {
|
||||
recorded_dao_in(|_, _| {}, edit)
|
||||
}
|
||||
|
||||
/// `recorded_dao` with `corpus_edit` run over the corpus and entities first.
|
||||
fn recorded_dao_in(
|
||||
corpus_edit: impl FnOnce(&mut CorpusStore, &weksa::EntityStore),
|
||||
edit: impl FnOnce(&mut HashMap<String, String>),
|
||||
) -> (Dao, RoundRecord) {
|
||||
let mut dao = dao_over(|corpus, entities| {
|
||||
let entities = weksa::EntityStore::open(entities).unwrap();
|
||||
let mut store = CorpusStore::open(corpus).unwrap();
|
||||
corpus_edit(&mut store, &entities);
|
||||
let mut written = store.get("item:daodejing:2").unwrap().unwrap();
|
||||
written.id = WRITTEN.into();
|
||||
written.text = "Heaven endures through the long years without end.".into();
|
||||
|
|
@ -4112,6 +4121,28 @@ fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() {
|
|||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_stance_ablation_creates_no_folder_and_wipes_no_store() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
|
||||
// No round 99: no folder appears.
|
||||
assert!(matches!(
|
||||
dao.run.stance_ablation(99, &mut rec, &mut model, &Scorer::new(Bag)),
|
||||
Err(RunError::Malformed("round"))
|
||||
));
|
||||
assert!(!dao.run.ablation_dir(99).exists(), "a refused ablation makes no folder");
|
||||
// An earlier ablation of a refused n keeps its stores byte for byte.
|
||||
let kept = dao.run.ablation_dir(2);
|
||||
std::fs::create_dir_all(&kept).unwrap();
|
||||
std::fs::write(kept.join("base.cc"), b"earlier ablation").unwrap();
|
||||
assert!(matches!(
|
||||
dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)),
|
||||
Err(RunError::Malformed("round"))
|
||||
));
|
||||
assert_eq!(std::fs::read(kept.join("base.cc")).unwrap(), b"earlier ablation");
|
||||
assert_eq!(std::fs::read_dir(&kept).unwrap().count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stance_ablation_refuses_a_cycle_round() {
|
||||
let mut dao = dao();
|
||||
|
|
@ -5299,3 +5330,408 @@ fn the_cycles_candidate_round_stops_on_a_withheld_lowering_prompt() {
|
|||
"the refused attempt's packets are discarded"
|
||||
);
|
||||
}
|
||||
|
||||
// ---- distil-slice: relower, a fixed-packet re-lowering measure ----
|
||||
|
||||
use weksa::run::{RelowerOutcome, RelowerReading};
|
||||
|
||||
/// `relower` of round 1 with the Bag scorer; `floor` is an optional second arm.
|
||||
fn relower(
|
||||
dao: &Dao,
|
||||
recorded: &mut dyn ModelPort,
|
||||
arm: &mut dyn ModelPort,
|
||||
floor: Option<&mut dyn ModelPort>,
|
||||
) -> Result<RelowerOutcome, RunError> {
|
||||
dao.run.relower(1, "arm", recorded, arm, floor, &Scorer::new(Bag))
|
||||
}
|
||||
|
||||
fn relowered(outcome: Result<RelowerOutcome, RunError>) -> (Vec<RelowerReading>, Option<weksa::run::Verdict>) {
|
||||
match outcome.unwrap() {
|
||||
RelowerOutcome::Read { languages, verdict } => (languages, verdict),
|
||||
RelowerOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_asks_only_validation_lowering_prompts_of_the_round() {
|
||||
let (dao, r) = recorded_dao(|_| {});
|
||||
let round_digests: Vec<String> = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect();
|
||||
// The reference pass: a lowerer shown every packet of the round, to learn
|
||||
// which digest is which item's.
|
||||
let (mut every, mut rec, mut arm) = (Lowerer::new(reader), Lowerer::new(reader), Lowerer::new(reader));
|
||||
read(ablate(&dao, &mut every, &mut Lowerer::new(reader)).unwrap());
|
||||
relowered(relower(&dao, &mut rec, &mut arm, None));
|
||||
let of = |keys: &[&str]| -> Vec<String> {
|
||||
every
|
||||
.seen
|
||||
.iter()
|
||||
.filter(|s| keys.contains(&s.1.as_str()))
|
||||
.map(|s| s.0.clone())
|
||||
.fold(Vec::new(), |mut v, d| {
|
||||
if !v.contains(&d) {
|
||||
v.push(d);
|
||||
}
|
||||
v
|
||||
})
|
||||
};
|
||||
let mut want = of(&["item:daodejing:2", "item:daodejing:3", WRITTEN]);
|
||||
want.sort();
|
||||
let mut got = arm.digests();
|
||||
got.sort();
|
||||
assert_eq!(got, want, "exactly the Validation lowering prompts");
|
||||
assert_eq!(want.len(), 3);
|
||||
for d in &want {
|
||||
assert!(round_digests.contains(d), "byte-identical to the round's prompt {d}");
|
||||
}
|
||||
// Neither the lone Train item, a Test item nor an encode prompt.
|
||||
for (_, key, text) in &arm.seen {
|
||||
assert!(!["item:daodejing:1", "item:daodejing:4", "item:daodejing:5"].contains(&key.as_str()));
|
||||
assert!(text.starts_with(LOWER_INSTRUCTIONS));
|
||||
}
|
||||
assert_eq!(rec.digests().len(), 3, "the recorded port is asked the same prompts");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_reads_recorded_and_arm_against_the_item() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
// An arm that says the item's own sentence beats a recorded lowering that
|
||||
// adds the stance to it.
|
||||
let (langs, verdict) = relowered(relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(reader),
|
||||
&mut Lowerer::new(blind),
|
||||
None,
|
||||
));
|
||||
let l = &langs[0];
|
||||
assert_eq!((l.language.as_str(), l.n, l.unusable), ("en-US", 3, 0));
|
||||
assert!(l.recorded.chrf < l.arm.chrf - 5.0, "{}", l.line());
|
||||
assert!(l.arm.composite > l.recorded.composite);
|
||||
assert!(l.floor.is_none() && verdict.is_none(), "no floor arm, no verdict");
|
||||
// An arm equal to the recorded answers reads equal.
|
||||
let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), None));
|
||||
assert_eq!(langs[0].recorded, langs[0].arm);
|
||||
}
|
||||
|
||||
/// Answers `text` for every item but `bad`, which gets something that is no lowering.
|
||||
struct Breaks(&'static str);
|
||||
|
||||
impl ModelPort for Breaks {
|
||||
fn model(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
|
||||
let key = prompt.item_keys()[0].clone();
|
||||
if key == self.0 {
|
||||
return Ok("this is not a lowering draft".into());
|
||||
}
|
||||
Ok(serde_json::json!({ "text": english(&key) }).to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_counts_an_unusable_arm_answer_as_zero() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let keys = ["item:daodejing:2", "item:daodejing:3", WRITTEN];
|
||||
let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None));
|
||||
let whole = whole[0].arm;
|
||||
let (mut chrf, mut meaning) = (0.0, 0.0);
|
||||
for key in keys {
|
||||
let (langs, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks(key), None));
|
||||
let l = &langs[0];
|
||||
assert_eq!((l.n, l.unusable), (3, 1), "{}", l.line());
|
||||
assert!(l.arm.chrf < whole.chrf && l.arm.meaning < whole.meaning, "{}", l.line());
|
||||
chrf += l.arm.chrf;
|
||||
meaning += l.arm.meaning;
|
||||
}
|
||||
// Each item is refused once: the three means hold every item twice at its
|
||||
// score and once at 0, so the refused item entered its mean as 0 and n stayed 3.
|
||||
assert!((chrf - 2.0 * whole.chrf).abs() < 1e-6, "{chrf} {}", whole.chrf);
|
||||
assert!((meaning - 2.0 * whole.meaning).abs() < 1e-6, "{meaning} {}", whole.meaning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_scores_an_unusable_recorded_answer_as_zero_without_counting_it_against_the_arm() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (whole, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Lowerer::new(blind), None));
|
||||
let (langs, _) = relowered(relower(&dao, &mut Breaks("item:daodejing:3"), &mut Lowerer::new(blind), None));
|
||||
let l = &langs[0];
|
||||
assert_eq!(l.unusable, 0, "{}", l.line());
|
||||
assert_eq!(l.arm, whole[0].arm);
|
||||
assert!(l.recorded.chrf < whole[0].recorded.chrf && l.recorded.meaning < whole[0].recorded.meaning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_decides_by_decide_panel_against_the_floor_arm() {
|
||||
use weksa::run::decide_panel;
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let run = |recorded: fn(&str, &str) -> String, arm: fn(&str, &str) -> String| {
|
||||
let mut floor = Lowerer::new(recorded);
|
||||
let (langs, verdict) = relowered(relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(recorded),
|
||||
&mut Lowerer::new(arm),
|
||||
Some(&mut floor),
|
||||
));
|
||||
let l = &langs[0];
|
||||
let floor_reading = l.floor.expect("a floor arm gives a floor");
|
||||
// A floor arm that equals the recorded answers has no spread.
|
||||
assert_eq!((floor_reading.chrf, floor_reading.meaning, floor_reading.composite), (0.0, 0.0, 0.0));
|
||||
assert_eq!(verdict, Some(decide_panel(&[(l.recorded, l.arm, floor_reading)])));
|
||||
verdict.unwrap()
|
||||
};
|
||||
assert_eq!(run(reader, reader), Verdict::Rejected(Why::NoGain));
|
||||
assert_eq!(run(reader, blind), Verdict::Accepted);
|
||||
assert!(matches!(run(blind, reader), Verdict::Rejected(Why::Regressed(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_floor_is_the_spread_of_recorded_and_the_floor_arm() {
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let (langs, _) = relowered(relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(blind),
|
||||
&mut Lowerer::new(blind),
|
||||
Some(&mut Breaks("item:daodejing:3")),
|
||||
));
|
||||
let l = &langs[0];
|
||||
let f = l.floor.unwrap();
|
||||
// The floor arm read as an arm against the same recorded answers.
|
||||
let (as_arm, _) = relowered(relower(&dao, &mut Lowerer::new(blind), &mut Breaks("item:daodejing:3"), None));
|
||||
let a = as_arm[0].arm;
|
||||
assert!(f.chrf > 0.0 && f.meaning > 0.0 && f.composite > 0.0);
|
||||
assert!((f.chrf - (l.recorded.chrf - a.chrf).abs()).abs() < 1e-9, "{}", l.line());
|
||||
assert!((f.composite - (l.recorded.composite - a.composite).abs()).abs() < 1e-9);
|
||||
assert!(l.line().contains(&format!("| floor chrf={:.2}", f.chrf)), "{}", l.line());
|
||||
assert_eq!(l.unusable, 0, "unusable counts the arm, not the floor");
|
||||
assert_eq!(l.floor_unusable, 1, "the floor arm's refusal is counted apart");
|
||||
assert!(l.line().contains(" unusable=0 recorded "), "{}", l.line());
|
||||
assert!(l.line().ends_with(" unusable=1"), "the floor's refusal is on the line: {}", l.line());
|
||||
// Without a floor arm there is none to count.
|
||||
assert_eq!(as_arm[0].floor_unusable, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_skips_and_counts_an_item_the_round_refused_before_admission() {
|
||||
// Validation item 3 is not an encodable draft in the round: it has no packet.
|
||||
let (dao, _) = recorded_dao(|answers| {
|
||||
answers.insert("item:daodejing:3".into(), "not a draft".into());
|
||||
});
|
||||
let (mut rec, mut arm) = (Lowerer::new(blind), Lowerer::new(blind));
|
||||
let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None));
|
||||
let l = &langs[0];
|
||||
assert_eq!((l.n, l.skipped, l.unusable), (2, 1, 0), "{}", l.line());
|
||||
assert!(l.line().starts_with("relower en-US n=2 skipped=1 unusable=0 "), "{}", l.line());
|
||||
assert_eq!(l.recorded, l.arm, "both readings are over the same two items");
|
||||
assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:3"), "nothing is asked for it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_skips_and_counts_an_item_whose_lowering_prompt_the_round_refused() {
|
||||
// ja-JP item 12 cites an entity with no ja form: the round admitted it and refused it
|
||||
// at Lower. Item 13 cites none and lowers.
|
||||
let mut dao = bilingual(None);
|
||||
let mut model = bilingual_fake();
|
||||
model.answers.insert("item:daodejing:12".into(), citing_laozi(2));
|
||||
model.answers.insert("item:daodejing:13".into(), json(&draft_of(3)));
|
||||
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
|
||||
assert_eq!(r.language("ja-JP").unwrap().validation_refused.lower, 1);
|
||||
let (mut rec, mut arm) = (Lowerer::new(bi_blind), Lowerer::new(bi_blind));
|
||||
let (langs, _) = relowered(relower(&dao, &mut rec, &mut arm, None));
|
||||
let ja = langs.iter().find(|l| l.language == "ja-JP").unwrap();
|
||||
assert_eq!((ja.n, ja.skipped), (1, 1), "{}", ja.line());
|
||||
assert_eq!(ja.recorded, ja.arm);
|
||||
let en = langs.iter().find(|l| l.language == "en-US").unwrap();
|
||||
assert_eq!((en.n, en.skipped), (2, 0), "{}", en.line());
|
||||
assert!(arm.seen.iter().all(|(_, key, _)| key != "item:daodejing:12"));
|
||||
}
|
||||
|
||||
/// The ja-JP item's own text, else the English one: `f` for en-US.
|
||||
fn bi(key: &str, text: &str, f: fn(&str, &str) -> String) -> String {
|
||||
match JA_TEXTS.iter().find(|(k, ..)| *k == key) {
|
||||
Some((_, ja, _)) => ja.to_string(),
|
||||
None => f(key, text),
|
||||
}
|
||||
}
|
||||
|
||||
fn bi_blind(key: &str, text: &str) -> String {
|
||||
bi(key, text, blind)
|
||||
}
|
||||
|
||||
fn bi_reader(key: &str, text: &str) -> String {
|
||||
bi(key, text, reader)
|
||||
}
|
||||
|
||||
/// Fine in English, nonsense in ja-JP.
|
||||
fn bi_ja_lost(key: &str, text: &str) -> String {
|
||||
match JA_TEXTS.iter().any(|(k, ..)| *k == key) {
|
||||
true => "zzzz".into(),
|
||||
false => blind(key, text),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_decides_over_the_whole_panel_not_its_first_language() {
|
||||
use weksa::run::decide_panel;
|
||||
let mut dao = bilingual(LAOZI_JA);
|
||||
recorded(round(&mut dao.run, &mut bilingual_fake(), &Scorer::new(Bag)).unwrap());
|
||||
// en-US gains (the arm drops the stance the recorded lowering repeats); ja-JP is lost.
|
||||
let (langs, verdict) = relowered(relower(
|
||||
&dao,
|
||||
&mut Lowerer::new(bi_reader),
|
||||
&mut Lowerer::new(bi_ja_lost),
|
||||
Some(&mut Lowerer::new(bi_reader)),
|
||||
));
|
||||
assert_eq!(
|
||||
langs.iter().map(|l| (l.language.as_str(), l.n)).collect::<Vec<_>>(),
|
||||
[("en-US", 2), ("ja-JP", 2)],
|
||||
"panel order, Validation only"
|
||||
);
|
||||
assert!(langs[0].arm.composite > langs[0].recorded.composite, "en-US gains");
|
||||
assert!(langs[1].arm.composite < langs[1].recorded.composite, "ja-JP is lost");
|
||||
let panel: Vec<_> = langs.iter().map(|l| (l.recorded, l.arm, l.floor.unwrap())).collect();
|
||||
assert_eq!(verdict, Some(decide_panel(&panel)));
|
||||
assert!(matches!(verdict, Some(Verdict::Rejected(Why::Regressed(_)))), "{verdict:?}");
|
||||
// The first language alone would have accepted.
|
||||
assert_eq!(decide_panel(&panel[..1]), Verdict::Accepted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_refuses_a_cycle_round_and_a_missing_recorded_answer() {
|
||||
use weksa::model::RecordedAnswers;
|
||||
let mut dao = dao();
|
||||
two_rounds(&mut dao);
|
||||
run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT));
|
||||
let candidate = dao.run.rounds().unwrap().into_iter().find(|r| r.cycle().is_some()).unwrap().n();
|
||||
let mut arm = Lowerer::new(reader);
|
||||
let r = dao.run.relower(candidate, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag));
|
||||
assert!(matches!(r, Err(RunError::Malformed("round"))));
|
||||
let r = dao.run.relower(99, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(Bag));
|
||||
assert!(matches!(r, Err(RunError::Malformed("round"))));
|
||||
assert!(arm.seen.is_empty());
|
||||
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let mut empty = RecordedAnswers::new(dao.root.join("round-1/nowhere"), "fake-model".into());
|
||||
let mut arm = Lowerer::new(reader);
|
||||
let r = relower(&dao, &mut empty, &mut arm, None);
|
||||
assert!(matches!(r, Err(RunError::Unrecorded)));
|
||||
assert!(arm.seen.is_empty(), "nothing is asked of the arm before the round's answer exists");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_floor_arm_with_an_unanswered_prompt_is_refused() {
|
||||
use weksa::model::RecordedAnswers;
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let mut floor = RecordedAnswers::new(dao.root.join("relower-1-floor"), "fake-model".into());
|
||||
let r = relower(&dao, &mut Lowerer::new(reader), &mut Lowerer::new(reader), Some(&mut floor));
|
||||
assert!(matches!(r, Err(RunError::FloorUnanswered)));
|
||||
assert!(!dao.root.join("relower-1-floor").exists(), "a floor arm is never written to");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_checks_the_embedder_not_the_model() {
|
||||
struct Named(Lowerer);
|
||||
impl ModelPort for Named {
|
||||
fn model(&self) -> &str {
|
||||
"another-model"
|
||||
}
|
||||
fn answer(&mut self, p: &Prompt) -> Result<String, PortError> {
|
||||
self.0.answer(p)
|
||||
}
|
||||
}
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let mut other = Named(Lowerer::new(blind));
|
||||
relowered(relower(&dao, &mut Lowerer::new(reader), &mut other, None));
|
||||
let mut arm = Lowerer::new(reader);
|
||||
let r = dao.run.relower(1, "arm", &mut Lowerer::new(reader), &mut arm, None, &Scorer::new(OtherEmbedder));
|
||||
assert!(matches!(r, Err(RunError::Drift(DriftOf::Embedder))));
|
||||
assert!(arm.seen.is_empty());
|
||||
}
|
||||
|
||||
/// Every file under `dir` with its bytes, by relative path.
|
||||
fn tree(dir: &Path) -> std::collections::BTreeMap<String, Vec<u8>> {
|
||||
let mut out = std::collections::BTreeMap::new();
|
||||
let mut stack = vec![dir.to_path_buf()];
|
||||
while let Some(d) = stack.pop() {
|
||||
for e in std::fs::read_dir(d).unwrap() {
|
||||
let p = e.unwrap().path();
|
||||
if p.is_dir() {
|
||||
stack.push(p);
|
||||
} else {
|
||||
out.insert(
|
||||
p.strip_prefix(dir).unwrap().to_string_lossy().into_owned(),
|
||||
std::fs::read(&p).unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_records_nothing() {
|
||||
use weksa::model::RecordedAnswers;
|
||||
let (dao, _) = recorded_dao(|_| {});
|
||||
let before = tree(&dao.root);
|
||||
let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
|
||||
// An unanswered arm writes its prompts in its own directory only.
|
||||
let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap();
|
||||
// The recorded answers are files here, as the relay leaves them.
|
||||
let r = relower(&dao, &mut recorded, &mut files, None);
|
||||
assert!(matches!(r, Err(RunError::Unrecorded)), "no round answer files yet");
|
||||
let after = tree(&dao.root);
|
||||
for (path, bytes) in &before {
|
||||
assert_eq!(after.get(path), Some(bytes), "{path} changed");
|
||||
}
|
||||
assert!(
|
||||
after.keys().all(|p| before.contains_key(p) || p.starts_with("relower-1-arm/")),
|
||||
"only relower-1-arm gains files"
|
||||
);
|
||||
|
||||
// With the round's answers present, the arm is asked and its prompts land in relower-1-arm.
|
||||
let (dao, r) = recorded_dao(|_| {});
|
||||
for call in &r.calls()[4..] {
|
||||
std::fs::write(
|
||||
dao.root.join(format!("round-1/{}.answer", call.prompt)),
|
||||
r#"{"text":"The highest excellence is like the water."}"#,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let before = tree(&dao.root);
|
||||
let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
|
||||
let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap();
|
||||
let RelowerOutcome::Unanswered { prompts } = relower(&dao, &mut recorded, &mut files, None).unwrap() else {
|
||||
panic!("the arm is unanswered")
|
||||
};
|
||||
assert_eq!(prompts.len(), 3);
|
||||
let after = tree(&dao.root);
|
||||
for (path, bytes) in &before {
|
||||
assert_eq!(after.get(path), Some(bytes), "{path} changed");
|
||||
}
|
||||
let added: Vec<&String> = after.keys().filter(|p| !before.contains_key(*p)).collect();
|
||||
assert!(added.iter().all(|p| p.starts_with("relower-1-arm/")), "{added:?}");
|
||||
assert_eq!(added.iter().filter(|p| p.ends_with(".prompt")).count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relower_arm_prompt_holding_test_text_is_withheld() {
|
||||
// A Test sentence that a Validation lowering prompt holds: a line of the
|
||||
// lowering instructions, which every lowering prompt carries.
|
||||
let line = LOWER_INSTRUCTIONS.lines().find(|l| l.trim().chars().count() > 30).unwrap().trim().to_string();
|
||||
let (dao, _) = recorded_dao_in(
|
||||
|corpus, entities| {
|
||||
let mut held = corpus.get("item:daodejing:4").unwrap().unwrap();
|
||||
held.id = "item:held:9".into();
|
||||
held.text = line.clone();
|
||||
corpus.add(held, entities).unwrap();
|
||||
},
|
||||
|_| {},
|
||||
);
|
||||
let mut files = dao.run.answer_files(dao.run.relower_dir(1, "arm")).unwrap();
|
||||
let r = relower(&dao, &mut Lowerer::new(reader), &mut files, None);
|
||||
assert!(matches!(r, Err(RunError::Withheld)), "{:?}", r.map(|_| ()));
|
||||
let dir = dao.run.relower_dir(1, "arm");
|
||||
let prompts = std::fs::read_dir(&dir)
|
||||
.map(|d| d.filter(|e| e.as_ref().unwrap().file_name().to_string_lossy().ends_with(".prompt")).count())
|
||||
.unwrap_or(0);
|
||||
assert_eq!(prompts, 0, "no prompt file is written");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -355,6 +355,31 @@ checkpoint the test-split composite is within the noise floor of validation.
|
|||
The test split is read only at checkpoints. Self then records the freeze, and
|
||||
the training set is the whole corpus re-encoded under the frozen revision.
|
||||
|
||||
## Distil measure (`distil-slice`)
|
||||
|
||||
A lowerer is compared on round n's Validation packets with the prompts
|
||||
byte-identical to the round's, by `weksa loop relower ROOT --round N --arm ARM
|
||||
--lowerer LABEL [--floor ARM2]`. Every arm is answered through the one prompt
|
||||
door (`RunStores::answer_files`, the Test-text guard); a tuned model is a script
|
||||
that reads `<digest>.prompt` and writes `<digest>.answer` in
|
||||
`<ROOT>/relower-<N>-<ARM>`, not a new adapter. The recorded reading is the
|
||||
round's own lowering, read over the same items as the arm. An item the round
|
||||
refused before admission (no packet) or at the lowering prompt has no prompt to
|
||||
ask: it is skipped, so the readings exclude it, and it is counted as `skipped=`
|
||||
on the line; `n` is the items read, so `recorded` can differ from the round's
|
||||
own Validation reading, which scores those items 0. An arm answer that is not a
|
||||
usable lowering scores 0 and counts in `unusable=`; the same for the floor arm,
|
||||
counted after `floor ... unusable=`. The floor is the spread of the recorded
|
||||
lowering and a second prompted arm (lowering-only noise, no encoder noise). The
|
||||
verdict is `decide_panel` over (recorded, arm, floor): Accepted is the rule's
|
||||
name for a gain past the floor, Rejected(Regressed) for a loss past it,
|
||||
Rejected(NoGain) for neither. None is yet a finding about the arm. The rule is calibration-pending: a one-pair floor is noisy
|
||||
(a simulated prompted repeat reads NoGain only about 7-15% of the time, Regressed
|
||||
44-60%, Accepted 33-41%), so NoGain is not yet evidence of non-inferiority and
|
||||
is not yet the bar for cut 13; the calibration arms (a prompted repeat, a
|
||||
control) decide what the floor must be before the verdict is read as a ruling.
|
||||
Training pairs are Train only, at a frozen revision; the export is 12b.
|
||||
|
||||
## Lowerer first cut (`golf-loop-lowerer-cut`)
|
||||
|
||||
Authority map for the smallest lowerer, written before its code (Imagination
|
||||
|
|
@ -436,5 +461,5 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
|
|||
9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs.
|
||||
10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`.
|
||||
11. `convergence-freeze`: the convergence test, `Checkpoint`'s one door (built from the convergence window over the persisted round records), the freeze, the re-encode into training pairs. Waits on 7, 9 and 10.
|
||||
12. `distil`: fine-tune a lowerer on the frozen training set. Waits on 11 and a host question asked then.
|
||||
12. `distil`, three cuts. 12a `distil-slice` (relower and the prompted baseline; spec r1). 12b `distil-export`: Train-only prompt/completion JSONL at a frozen revision, Test-text guarded (follow-up `distil-export-at-freeze`); waits on 11. 12c `distil-train`: waits on 12b and the host question (follow-up `distil-host-question-owed`).
|
||||
13. `live-lowering`: agents emit packets with stance; Weksa reads speaker state from persona state (follow-up `persona-state-speech-community`) and lowers for the reader. Waits on 11.
|
||||
|
|
|
|||
Loading…
Reference in a new issue