Weksa/crates/weksa/tests/golf_loop.rs

4559 lines
169 KiB
Rust

//! The golf loop: a pinned run encodes Train and Validation through a
//! `ModelPort`, admits, lowers, scores, and keeps a typed round record.
mod common;
use std::path::{Path, PathBuf};
use common::{ent, run_over, world};
use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore};
use weksa::corpus::ModelFamily;
use weksa::run::{DriftOf, RunError, RunPaths, RunRecord, RunStores};
use weksa::split::fnv1a64;
/// Rewrites the stored run record, as only a test (or a stale file) can.
fn rewrite_run(root: &Path, change: impl FnOnce(&mut RunRecord)) {
let mut cache = CultCache::new();
cache.register_entry_type::<RunRecord>().unwrap();
cache
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
.unwrap();
cache.pull_all_backing_stores().unwrap();
let mut record = cache.get_all::<RunRecord>().unwrap().remove(0);
change(&mut record);
cache.put(format!("run:{}", record.label), &record).unwrap();
}
fn paths(dir: &Path) -> (PathBuf, PathBuf, PathBuf) {
(dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"))
}
fn hex(path: &Path) -> String {
format!("{:016x}", fnv1a64(&std::fs::read(path).unwrap()))
}
#[test]
fn a_run_pins_its_stores_and_refuses_drift() {
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let (c, i, e) = paths(&dir);
let at = |label: &str, model: &str| {
RunStores::create(
&root,
label,
RunPaths {
corpus: &c,
inventory: &i,
entities: &e,
},
model,
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
// Opening a root with no run makes none.
assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun));
assert!(!root.join("run.cc").exists(), "open must not create run.cc");
assert_eq!(
at("Bad Label", "m").err(),
Some(RunError::Malformed("label"))
);
assert_eq!(at("t", "").err(), Some(RunError::Malformed("model")));
assert!(
!root.join("run.cc").exists(),
"a refused create writes nothing"
);
let run = at("t", "fake-model").unwrap();
let r = run.record();
assert_eq!((r.label.as_str(), r.model.as_str()), ("t", "fake-model"));
assert_eq!(
r.corpus,
std::fs::canonicalize(&c).unwrap().to_str().unwrap()
);
assert_eq!(r.corpus_digest, hex(&c));
assert_eq!(r.entity_digest, hex(&e));
assert_eq!(r.chrf_version, weksa::score::CHRF_VERSION);
assert_eq!(at("u", "fake-model").err(), Some(RunError::Exists));
// The record is all open reads: the same run, from the same digests.
assert_eq!(RunStores::open(&root).unwrap().record(), run.record());
// A corpus written after the pin is another corpus.
let item = common::item_in(
"item:nibu:2",
"Another line.",
&common::group_in_bucket(10),
common::cast(),
);
w.corpus.add(item, &w.entities).unwrap();
assert_eq!(
RunStores::open(&root).err(),
Some(RunError::Drift(DriftOf::Corpus))
);
}
#[test]
fn open_checks_each_pin_by_name() {
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let drift = |change: &dyn Fn(&mut RunRecord)| {
let _ = std::fs::remove_dir_all(&root);
run_over(&dir);
rewrite_run(&root, change);
RunStores::open(&root).err()
};
assert_eq!(
drift(&|r| r.prompt_digest = "0".repeat(16)),
Some(RunError::Drift(DriftOf::Prompts))
);
assert_eq!(
drift(&|r| r.corpus_digest = "0".repeat(16)),
Some(RunError::Drift(DriftOf::Corpus))
);
assert_eq!(
drift(&|r| r.entity_digest = "0".repeat(16)),
Some(RunError::Drift(DriftOf::Entities))
);
assert_eq!(
drift(&|r| r.inventory_digest = "0".repeat(16)),
Some(RunError::Drift(DriftOf::Inventory))
);
assert_eq!(drift(&|_| {}), None, "the unchanged record opens");
// An entity put after the pin is another entity store.
let _ = std::fs::remove_dir_all(&root);
run_over(&dir);
w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
assert_eq!(
RunStores::open(&root).err(),
Some(RunError::Drift(DriftOf::Entities))
);
}
#[test]
fn a_run_works_on_its_own_copy_of_the_inventory() {
let w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let (c, i, e) = paths(&dir);
let source_bytes = std::fs::read(&i).unwrap();
let mut run = RunStores::create(
&root,
"t",
RunPaths {
corpus: &c,
inventory: &i,
entities: &e,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
// The copy is byte for byte the source, and the record pins its digest.
let copy = root.join("inventory.cc");
assert_eq!(std::fs::read(&copy).unwrap(), source_bytes);
assert_eq!(run.record().inventory_digest, hex(&i));
assert_eq!(run.record().inventory_digest, hex(&copy));
assert_eq!(
run.record().inventory,
std::fs::canonicalize(&i).unwrap().to_str().unwrap(),
"the source path is provenance"
);
let rev = run.inventory().inventory().unwrap().latest().unwrap().n;
// A revision written to the source after open is a revision the run never
// reads, and it does not drift the run.
let mut source = weksa::InventoryStore::open(&i).unwrap();
source
.admit_revision(
weksa::InventoryRevision {
n: rev + 1,
parent: Some(rev),
changes: vec![],
frozen: false,
},
vec![],
)
.unwrap();
assert_ne!(hex(&i), run.record().inventory_digest);
run = RunStores::open(&root).unwrap();
assert_eq!(
run.inventory().inventory().unwrap().latest().unwrap().n,
rev,
"the run reads its copy"
);
assert_eq!(std::fs::read(&copy).unwrap(), source_bytes);
// Even with the source gone the run opens.
std::fs::remove_file(&i).unwrap();
assert!(RunStores::open(&root).is_ok());
// A revision appended to the copy is drift.
let mut head = weksa::InventoryStore::open(&copy).unwrap();
head.admit_revision(
weksa::InventoryRevision {
n: rev + 1,
parent: Some(rev),
changes: vec![],
frozen: false,
},
vec![],
)
.unwrap();
assert_eq!(
RunStores::open(&root).err(),
Some(RunError::Drift(DriftOf::Inventory))
);
}
#[test]
fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() {
let dao = dao();
let bytes = std::fs::read(&dao.inventory).unwrap();
let create = |root: &Path, inventory: &Path| {
RunStores::create(
root,
"x",
RunPaths {
corpus: &dao.corpus,
inventory,
entities: &dao.entities,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
// The source is the root's own inventory.cc: the copy would open it as its
// destination. It is refused and the source bytes survive.
let own = dao.root.join("own");
std::fs::create_dir_all(&own).unwrap();
std::fs::write(own.join("inventory.cc"), &bytes).unwrap();
assert_eq!(
create(&own, &own.join("inventory.cc")).err(),
Some(RunError::Io(std::io::ErrorKind::AlreadyExists))
);
assert_eq!(std::fs::read(own.join("inventory.cc")).unwrap(), bytes);
// Any other file of that name in the root is refused untouched, too.
let other = dao.root.join("other");
std::fs::create_dir_all(&other).unwrap();
std::fs::write(other.join("inventory.cc"), b"not yours").unwrap();
assert_eq!(
create(&other, &dao.inventory).err(),
Some(RunError::Io(std::io::ErrorKind::AlreadyExists))
);
assert_eq!(
std::fs::read(other.join("inventory.cc")).unwrap(),
b"not yours"
);
assert_eq!(std::fs::read(&dao.inventory).unwrap(), bytes);
}
#[test]
fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() {
let dao = dao();
let root = dao.root.join("retry");
std::fs::create_dir_all(&root).unwrap();
let create = |inventory: &Path, entities: &Path| {
RunStores::create(
&root,
"x",
RunPaths {
corpus: &dao.corpus,
inventory,
entities,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
};
// A directory opens for reading and fails mid-copy, after the destination exists.
let dir = dao.root.join("a-directory");
std::fs::create_dir_all(&dir).unwrap();
assert!(create(&dir, &dao.entities).is_err());
assert!(!root.join("inventory.cc").exists());
// A directory as the entities store fails after the inventory was copied.
assert!(create(&dao.inventory, &dir).is_err());
assert!(!root.join("inventory.cc").exists());
assert!(create(&dao.inventory, &dao.entities).is_ok());
}
/// `weksa.loop_run` as it was before the run kept its own inventory copy: keys
/// 0 to 10.
#[derive(Clone, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_run")]
struct LegacyRun {
#[cultcache(key = 0)]
label: String,
#[cultcache(key = 1)]
corpus: String,
#[cultcache(key = 2)]
inventory: String,
#[cultcache(key = 3)]
entities: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
prompt_digest: String,
#[cultcache(key = 7)]
lowerer_digest: String,
#[cultcache(key = 8)]
model: String,
#[cultcache(key = 9)]
embedder: String,
#[cultcache(key = 10)]
chrf_version: String,
}
#[test]
fn a_run_from_before_the_copy_does_not_open() {
// A run.cc without key 11 is a scratch run off git: its open fails Store.
let w = world();
let dir = w.dir.path().to_path_buf();
let current = run_over(&dir).record().clone();
let root = dir.join("run");
std::fs::remove_file(root.join("run.cc")).unwrap();
let mut cache = CultCache::new();
cache.register_entry_type::<LegacyRun>().unwrap();
cache
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
.unwrap();
cache
.put(
"run:t",
&LegacyRun {
label: current.label,
corpus: current.corpus,
inventory: current.inventory,
entities: current.entities,
corpus_digest: current.corpus_digest,
entity_digest: current.entity_digest,
prompt_digest: current.prompt_digest,
lowerer_digest: String::new(),
model: current.model,
embedder: current.embedder,
chrf_version: current.chrf_version,
},
)
.unwrap();
assert_eq!(RunStores::open(&root).err(), Some(RunError::Store));
}
#[test]
fn a_store_changed_between_its_digest_and_its_open_is_drift() {
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
run_over(&dir);
let more = |w: &mut common::World| {
let item = common::item_in(
"item:nibu:2",
"Another line.",
&common::group_in_bucket(10),
common::cast(),
);
w.corpus.add(item, &w.entities).unwrap();
};
// The pins hold at the check; the corpus changes before the stores open.
assert_eq!(
RunStores::open_between(&root, || more(&mut w)).err(),
Some(RunError::Drift(DriftOf::Corpus))
);
// The same window at create: nothing is pinned and the inventory copy and its
// lock are taken away again, so the retry creates.
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
for name in ["c.cc", "i.cc", "entities.cc"] {
assert!(dir.join(name).exists(), "{name}");
}
let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"));
let paths = || weksa::run::RunPaths {
corpus: &c,
inventory: &i,
entities: &e,
};
assert_eq!(
RunStores::create_between(
&root,
"t",
paths(),
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
|| more(&mut w)
)
.err(),
Some(RunError::Drift(DriftOf::Corpus))
);
assert!(!root.join("inventory.cc").exists());
// Not the copy and not the lock the store's open left beside it.
assert!(!root.join("inventory.cc.lock").exists());
assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun));
RunStores::create(
&root,
"t",
paths(),
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
}
/// Runs `change` in the window between the pins and the open, at `open` and at
/// `create`, and returns what each refused with plus what `create` left behind.
fn window(
change: impl Fn(&mut common::World, &Path),
) -> (Option<RunError>, Option<RunError>, bool, Option<RunError>) {
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
run_over(&dir);
let opened = RunStores::open_between(&root, || change(&mut w, &root)).err();
let mut w = world();
let dir = w.dir.path().to_path_buf();
let root = dir.join("run");
let (c, i, e) = (dir.join("c.cc"), dir.join("i.cc"), dir.join("entities.cc"));
let paths = weksa::run::RunPaths {
corpus: &c,
inventory: &i,
entities: &e,
};
let created = RunStores::create_between(
&root,
"t",
paths,
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
|| change(&mut w, &root),
)
.err();
let left = root.join("inventory.cc").exists() || root.join("inventory.cc.lock").exists();
(opened, created, left, RunStores::open(&root).err())
}
#[test]
fn an_entity_store_changed_between_its_digest_and_its_open_is_drift() {
let (opened, created, left, after) = window(|w, _| {
w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
});
assert_eq!(opened, Some(RunError::Drift(DriftOf::Entities)));
assert_eq!(created, Some(RunError::Drift(DriftOf::Entities)));
assert!(!left);
assert_eq!(after, Some(RunError::NoRun));
}
#[test]
fn a_head_inventory_changed_between_its_digest_and_its_open_is_drift() {
let (opened, created, left, after) = window(|_, root| {
weksa::InventoryStore::open(&root.join("inventory.cc"))
.unwrap()
.admit_revision(
weksa::InventoryRevision {
n: 5,
parent: Some(4),
changes: vec![],
frozen: false,
},
vec![],
)
.unwrap();
});
assert_eq!(opened, Some(RunError::Drift(DriftOf::Inventory)));
assert_eq!(created, Some(RunError::Drift(DriftOf::Inventory)));
assert!(!left);
assert_eq!(after, Some(RunError::NoRun));
}
#[test]
fn source_digests_cover_their_files() {
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let read = |name: &str| std::fs::read(src.join(name)).unwrap();
let digest = |name: &str| fnv1a64(&read(name));
// The prompt's partition and names come from split.rs, and the lowering
// prompt reads lower.rs's target language and entity port and prints
// packet.rs's serde shapes: all four are covered.
let (context, split, lower, packet) = (
read("context.rs"),
read("split.rs"),
read("lower.rs"),
read("packet.rs"),
);
// Each file enters behind its length, so the parts have one framing.
let framed = [
(context.len() as u64).to_le_bytes().as_slice(),
&context,
(split.len() as u64).to_le_bytes().as_slice(),
&split,
(lower.len() as u64).to_le_bytes().as_slice(),
&lower,
(packet.len() as u64).to_le_bytes().as_slice(),
&packet,
]
.concat();
assert_eq!(weksa::run::PROMPT_SOURCE_DIGEST, fnv1a64(&framed));
assert_ne!(
weksa::run::PROMPT_SOURCE_DIGEST,
fnv1a64(&[context.clone(), split.clone(), lower.clone(), packet.clone()].concat()),
"bare concatenation is not the digest"
);
// Moving the boundary between the parts moves the digest.
assert_ne!(
weksa::split::fnv1a64_of(&[b"ab", b"c"]),
weksa::split::fnv1a64_of(&[b"a", b"bc"])
);
assert_ne!(weksa::run::PROMPT_SOURCE_DIGEST, digest("context.rs"));
}
/// The encode prompt of `item:nibu:1` over `world()` at revision 4.
fn prompt() -> weksa::context::Prompt {
let w = world();
let run = run_over(w.dir.path());
let inv = run.inventory().inventory().unwrap();
weksa::context::Prompt::encode(
&run.offer("item:nibu:1").unwrap().unwrap(),
&run.train().unwrap(),
&inv,
4,
)
}
#[test]
fn encode_prompt_lists_the_live_inventory() {
let w = world();
let run = run_over(w.dir.path());
let inv = run.inventory().inventory().unwrap();
let train = run.train().unwrap();
let offer = run.offer("item:nibu:1").unwrap().unwrap();
let at = |rev| weksa::context::Prompt::encode(&offer, &train, &inv, rev);
let listed = |text: &str| -> Vec<String> {
let (_, tail) = text.split_once("\n\ninventory rev:").unwrap();
tail.lines().skip(1).map(str::to_string).collect()
};
let ids = |lines: &[String]| -> Vec<String> {
lines
.iter()
.map(|l| l.split(':').take(2).collect::<Vec<_>>().join(":"))
.collect()
};
let p4 = at(4);
assert!(p4.text().contains("\n\ninventory rev:4\n"));
let lines4 = listed(p4.text());
let live4: Vec<String> = inv
.entries()
.filter(|e| e.live_at(4))
.map(|e| e.id.to_string())
.collect();
assert_eq!(ids(&lines4), live4, "every live entry, in id order");
assert!(live4.contains(&common::CONCEPT_LATE.to_string()));
assert!(
!live4.contains(&common::CONCEPT_RETIRED.to_string()),
"retired at 3"
);
let ids2 = ids(&listed(at(2).text()));
assert!(
ids2.contains(&common::CONCEPT_RETIRED.to_string()),
"live at 2"
);
assert!(
!ids2.contains(&common::CONCEPT_LATE.to_string()),
"not yet introduced"
);
// A line carries the gloss; a feature its kind and slots, an options feature its labels.
let line = |id: &str| {
lines4
.iter()
.find(|l| l.starts_with(&format!("{id}: ")))
.unwrap()
.clone()
};
assert_eq!(
line(common::CONCEPT_A),
format!("{}: {}", common::CONCEPT_A, common::CONCEPT_A)
);
assert!(
line(common::HEDGE).contains("[grade; slots "),
"{}",
line(common::HEDGE)
);
assert!(
line(common::MODALITY).contains("[options "),
"{}",
line(common::MODALITY)
);
assert!(
line(common::MODALITY).contains("potential"),
"{}",
line(common::MODALITY)
);
assert!(
line(common::EXT).ends_with("[grade]"),
"an extension has a kind and no slots: {}",
line(common::EXT)
);
assert!(
at(4)
.text()
.starts_with(weksa::context::ENCODE_INSTRUCTIONS)
);
assert_eq!(at(4).digest().len(), 16);
assert_eq!(at(4).digest(), p4.digest());
assert_ne!(at(4).digest(), at(2).digest());
assert_eq!(
p4.digest(),
format!("{:016x}", fnv1a64(p4.text().as_bytes()))
);
}
#[test]
fn answer_files_write_the_prompt_and_read_the_answer() {
use std::io::ErrorKind;
use weksa::model::{AnswerFiles, ModelPort, PortError};
let p = prompt();
let dir = tempfile::tempdir().unwrap();
let round = dir.path().join("round-1");
let mut files = AnswerFiles::new(round.clone(), "answerer".into());
assert_eq!(files.model(), "answerer");
// No answer: the prompt is written under its digest, and nothing else.
assert_eq!(files.answer(&p), Err(PortError::Unanswered));
let prompt_file = round.join(format!("{}.prompt", p.digest()));
assert_eq!(std::fs::read_to_string(&prompt_file).unwrap(), p.text());
assert_eq!(std::fs::read_dir(&round).unwrap().count(), 1);
// Asking again leaves the equal file alone.
let before = std::fs::metadata(&prompt_file).unwrap().modified().unwrap();
assert_eq!(files.answer(&p), Err(PortError::Unanswered));
assert_eq!(
std::fs::metadata(&prompt_file).unwrap().modified().unwrap(),
before
);
// A different file under the digest is refused, not overwritten.
std::fs::write(&prompt_file, "other").unwrap();
assert_eq!(
files.answer(&p),
Err(PortError::Io(ErrorKind::AlreadyExists))
);
assert_eq!(std::fs::read_to_string(&prompt_file).unwrap(), "other");
// The answer is read as it was written, whitespace and all.
let answer = round.join(format!("{}.answer", p.digest()));
std::fs::write(&answer, " {\"x\": 1}\n").unwrap();
assert_eq!(files.answer(&p).unwrap(), " {\"x\": 1}\n");
std::fs::write(&answer, [0xff, 0xfe]).unwrap();
assert_eq!(files.answer(&p), Err(PortError::Io(ErrorKind::InvalidData)));
// A directory that cannot be made is an io error of its kind.
std::fs::write(dir.path().join("file"), "").unwrap();
let mut blocked = AnswerFiles::new(dir.path().join("file").join("round"), "m".into());
assert!(matches!(blocked.answer(&p), Err(PortError::Io(_))));
// A reply or prompt file that cannot be read is an io error, never "unanswered".
use std::os::unix::fs::PermissionsExt;
let unreadable = |path: &Path| {
std::fs::write(path, "x").unwrap();
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o200)).unwrap();
};
unreadable(&answer);
assert_eq!(
files.answer(&p),
Err(PortError::Io(ErrorKind::PermissionDenied))
);
std::fs::remove_file(&answer).unwrap();
unreadable(&prompt_file);
assert_eq!(
files.answer(&p),
Err(PortError::Io(ErrorKind::PermissionDenied))
);
std::fs::set_permissions(&prompt_file, std::fs::Permissions::from_mode(0o600)).unwrap();
assert_eq!(
std::fs::read_to_string(&prompt_file).unwrap(),
"x",
"an unreadable prompt file is not rewritten"
);
assert_eq!(
PortError::Unanswered.to_string(),
"the prompt is unanswered"
);
assert_eq!(
PortError::Io(ErrorKind::InvalidData).to_string(),
"io error InvalidData"
);
}
// ---- rounds over the Dao De Jing test bed ----
use std::collections::HashMap;
use common::Bag;
use weksa::context::{ENCODE_INSTRUCTIONS, LOWER_INSTRUCTIONS, MINT_INSTRUCTIONS, Prompt};
use weksa::daodejing::packets;
use weksa::model::{AnswerFiles, ModelPort, PortError};
use weksa::run::{Outcome, RoundOutcome, RoundRecord, Stage, round};
use weksa::{CorpusStore, Embedder, InventoryStore, PacketDraft, PacketStore, Scorer};
/// Answers by the item the prompt holds; panics on an item it was not given,
/// and records every prompt it was shown. A prompt that starts with
/// `MINT_INSTRUCTIONS` is the editor's and is answered from `edit` (a panic when
/// none was given). With `candidate` set, an encode prompt that lists
/// `inventory rev:<that revision>` is answered from the candidate answers.
struct FakeModel {
name: &'static str,
answers: HashMap<String, String>,
/// The lowering answered for an item: `{"text": lowered[item]}`; a panic on
/// an item with none.
lowered: HashMap<String, String>,
/// When set, the answer to every lowering prompt instead of `lowered`.
raw_lowering: Option<Result<String, PortError>>,
candidate: Option<(u32, HashMap<String, String>)>,
edit: Option<Result<String, PortError>>,
seen: Vec<(String, String)>,
mint_keys: Vec<Vec<String>>,
}
impl ModelPort for FakeModel {
fn model(&self) -> &str {
self.name
}
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
self.seen.push((prompt.digest(), prompt.text().to_string()));
if prompt.text().starts_with(MINT_INSTRUCTIONS) {
self.mint_keys.push(prompt.item_keys().to_vec());
return self.edit.clone().expect("the model was asked for an edit");
}
let key = prompt.item_keys()[0].clone();
if prompt.text().starts_with(LOWER_INSTRUCTIONS) {
if let Some(raw) = &self.raw_lowering {
return raw.clone();
}
let text = self
.lowered
.get(&key)
.unwrap_or_else(|| panic!("the model was asked to lower {key}"));
return Ok(serde_json::json!({ "text": text }).to_string());
}
let answers = match &self.candidate {
Some((rev, candidate))
if prompt
.text()
.contains(&format!("\n\ninventory rev:{rev}\n")) =>
{
candidate
}
_ => &self.answers,
};
match answers.get(&key) {
Some(answer) => Ok(answer.clone()),
None => panic!("the model was asked about {key}"),
}
}
}
fn draft_of(item: usize) -> PacketDraft {
PacketDraft::of(&packets()[item - 1])
}
fn json(draft: &PacketDraft) -> String {
serde_json::to_string(draft).unwrap()
}
fn keyed(answers: &[(usize, String)]) -> HashMap<String, String> {
answers
.iter()
.map(|(n, a)| (format!("item:daodejing:{n}"), a.clone()))
.collect()
}
/// The five English texts the deleted template table gave the Dao packets
/// (`tests/daodejing.rs`): the lowering the fake model answers, so the score
/// pins hold.
fn table_english() -> HashMap<String, String> {
[
"The sage empties the minds. The sage fills the bellies. The sage weakens the wills. The sage strengthens the bones.",
"The heaven is long-enduring. The earth continues long.",
"The highest excellence is like the water.",
"The words that are strictly true seem paradoxical.",
"The space between heaven and earth may be compared to the bellows.",
]
.iter()
.enumerate()
.map(|(i, text)| (format!("item:daodejing:{}", i + 1), (*text).to_string()))
.collect()
}
/// A fake that answers the Dao packets for the Train and Validation items.
fn fake(answers: &[(usize, String)]) -> FakeModel {
FakeModel {
name: "fake-model",
answers: keyed(answers),
lowered: table_english(),
raw_lowering: None,
candidate: None,
edit: None,
seen: vec![],
mint_keys: vec![],
}
}
fn good_answers() -> Vec<(usize, String)> {
(1..=3).map(|n| (n, json(&draft_of(n)))).collect()
}
struct Dao {
_dir: tempfile::TempDir,
root: PathBuf,
corpus: PathBuf,
inventory: PathBuf,
entities: PathBuf,
run: RunStores,
}
/// A run over copies of `data/daodejing/{corpus,inventory,entity}.cc`.
fn dao() -> Dao {
dao_over(|_, _| {})
}
/// A Dao run whose stores `edit` (given the corpus and entity paths) changed
/// before the run pinned them.
fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao {
dao_panel(edit, &["en-US"])
}
/// `dao_over` with the run opened on `panel`.
fn dao_panel(edit: impl FnOnce(&Path, &Path), panel: &[&str]) -> Dao {
let panel: Vec<String> = panel.iter().map(|t| t.to_string()).collect();
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing");
let dir = tempfile::tempdir().unwrap();
let copy = |from: &str, to: &str| {
std::fs::copy(src.join(from), dir.path().join(to)).unwrap();
dir.path().join(to)
};
let (corpus, inventory, entities) = (
copy("corpus.cc", "c.cc"),
copy("inventory.cc", "i.cc"),
copy("entity.cc", "e.cc"),
);
edit(&corpus, &entities);
let root = dir.path().join("run");
let run = RunStores::create(
&root,
"dao",
RunPaths {
corpus: &corpus,
inventory: &inventory,
entities: &entities,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&panel,
)
.unwrap();
Dao {
_dir: dir,
root,
corpus,
inventory,
entities,
run,
}
}
fn recorded(outcome: RoundOutcome) -> RoundRecord {
match outcome {
RoundOutcome::Recorded(r) => r,
RoundOutcome::Unanswered { prompts } => panic!("unanswered: {prompts:?}"),
}
}
fn two_dp(x: f64) -> f64 {
(x * 100.0).round() / 100.0
}
#[test]
fn round_over_the_dao_stores_scores_train_and_validation() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let pinned = dao.run.record().clone();
assert_eq!((r.n(), r.inventory_rev()), (1, 3));
let items: Vec<&[String]> = r.calls().iter().map(|c| c.items.as_slice()).collect();
assert_eq!(
items,
[
["item:daodejing:1"],
["item:daodejing:2"],
["item:daodejing:3"],
["item:daodejing:1"],
["item:daodejing:2"],
["item:daodejing:3"]
],
"the encode calls, then the lowering calls"
);
for (call, (digest, _)) in r.calls().iter().zip(&model.seen) {
assert_eq!(&call.prompt, digest);
}
// Train: a row per item, with the packet that was admitted and its score.
assert_eq!(r.train().len(), 1);
let row = &r.train()[0];
assert_eq!(row.item, "item:daodejing:1");
let Outcome::Scored { chrf, meaning } = row.outcome else {
panic!("{:?}", row.outcome)
};
assert_eq!(two_dp(chrf), 44.18);
assert!(meaning.is_some());
assert_eq!((r.language("en-US").unwrap().train_mean.count, r.language("en-US").unwrap().train_mean.chrf), (1, Some(chrf)));
// Validation: means and refusal counts only.
let v = &r.language("en-US").unwrap().validation_mean;
assert_eq!(v.count, 2);
assert!((v.chrf.unwrap() - 78.56).abs() < 0.01, "{:?}", v.chrf);
assert!(v.meaning.is_some());
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2));
// The record carries the run's pins.
assert_eq!(r.prompt_digest(), pinned.prompt_digest);
assert_eq!(r.corpus_digest(), pinned.corpus_digest);
assert_eq!(r.entity_digest(), pinned.entity_digest);
assert_eq!(r.model(), "fake-model");
assert_eq!(r.embedder(), "bag");
assert_eq!(r.chrf_version(), weksa::score::CHRF_VERSION);
assert_eq!(r.inventory_digest(), pinned.inventory_digest);
assert_eq!(r.cycle(), None, "a plain round belongs to no cycle");
// It is stored, and the packets are in the round's own store, made by the encoder step.
assert_eq!(
RunStores::open(&dao.root).unwrap().rounds().unwrap(),
std::slice::from_ref(&r)
);
let packets = PacketStore::open(&dao.root.join("round-1/packets.cc")).unwrap();
let all = packets.all().unwrap();
assert_eq!(all.len(), 3);
let encoder = weksa::encode::encoder_id("fake-model", &pinned.prompt_digest);
for p in &all {
assert_eq!(p.provenance.encoder, encoder);
assert_eq!(p.inventory_rev, 3);
assert_eq!(p.trace, Default::default());
}
assert_eq!(
row.packet.as_deref(),
Some(
all.iter()
.find(|p| p.provenance.corpus_item.as_str() == "item:daodejing:1")
.unwrap()
.id
.as_str()
)
);
assert!(!dao.root.join("round-2").exists());
}
#[test]
fn round_never_offers_or_records_test() {
let mut dao = dao();
// The fake panics if asked about item 4 or 5.
let mut model = fake(&good_answers());
recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert!(
model
.seen
.iter()
.all(|(_, t)| !t.contains("item:daodejing:4") && !t.contains("item:daodejing:5"))
);
let corpus = weksa::CorpusStore::open(&dao.corpus).unwrap();
let mut files = vec![dao.root.join("run.cc")];
for entry in std::fs::read_dir(dao.root.join("round-1")).unwrap() {
files.push(entry.unwrap().path());
}
let mut needles: Vec<String> = vec!["item:daodejing:4".into(), "item:daodejing:5".into()];
needles.extend(
["item:daodejing:4", "item:daodejing:5"].map(|k| corpus.get(k).unwrap().unwrap().text),
);
for file in files {
let bytes = std::fs::read(&file).unwrap();
for needle in &needles {
assert!(
!bytes.windows(needle.len()).any(|w| w == needle.as_bytes()),
"{} holds a Test key or text",
file.display()
);
}
}
// Nor does the record hold any Train or Validation text.
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
for k in ["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"] {
let text = corpus.get(k).unwrap().unwrap().text;
assert!(
!run_cc.windows(text.len()).any(|w| w == text.as_bytes()),
"{k} text in run.cc"
);
}
}
/// The key and prompt of every item a round offers, as a round would build them.
fn prompts(run: &RunStores) -> Vec<(String, Prompt)> {
let inv = run.inventory().inventory().unwrap();
let rev = inv.latest().unwrap().n;
let train = run.train().unwrap();
run.offers()
.unwrap()
.iter()
.map(|o| (o.key().to_string(), Prompt::encode(o, &train, &inv, rev)))
.collect()
}
/// The key and prompt of every lowering a round over `dao`'s run asks for when
/// every item encodes to its Dao packet and is admitted, as a round builds them.
fn lowering_prompts(dao: &Dao, inventory: &InventoryStore) -> Vec<(String, Prompt)> {
let corpus = CorpusStore::open(&dao.corpus).unwrap();
let dir = tempfile::tempdir().unwrap();
let mut store = PacketStore::open(&dir.path().join("p.cc")).unwrap();
let inv = inventory.inventory().unwrap();
(1..=3)
.map(|n| {
let mut packet = packets().remove(n - 1);
// The encoder step stamps the inventory's latest revision.
packet.inventory_rev = inv.latest().unwrap().n;
let key = packet.provenance.corpus_item.clone();
let view = store
.admit(packet, inventory, &corpus, dao.run.entities())
.unwrap();
let prompt = Prompt::lower(&view, &key, "en-US", &inv, dao.run.entities()).unwrap();
(key.to_string(), prompt)
})
.collect()
}
#[test]
fn unanswered_round_writes_nothing() {
let mut dao = dao();
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
let expected = prompts(&dao.run);
assert_eq!(expected.len(), 3);
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
panic!("answered from nothing")
};
let want: Vec<String> = expected.iter().map(|(_, p)| p.digest()).collect();
assert_eq!(digests, want);
let round1 = dao.root.join("round-1");
for (_, p) in &expected {
assert_eq!(
std::fs::read_to_string(round1.join(format!("{}.prompt", p.digest()))).unwrap(),
p.text()
);
}
assert_eq!(
std::fs::read_dir(&round1).unwrap().count(),
3,
"prompts only: no packets.cc"
);
assert!(dao.run.rounds().unwrap().is_empty());
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
// Two of three answers is still nothing written.
let answers: HashMap<String, String> = good_answers()
.into_iter()
.map(|(n, a)| (format!("item:daodejing:{n}"), a))
.collect();
for (key, p) in &expected[..2] {
std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap();
}
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
panic!("answered from two of three")
};
assert_eq!(digests, [want[2].clone()]);
assert!(!round1.join("packets.cc").exists());
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
// The last encode answer moves the round to its second phase: the lowering
// prompts are asked, and still nothing is written.
let (key, p) = &expected[2];
std::fs::write(round1.join(format!("{}.answer", p.digest())), &answers[key]).unwrap();
let lowering = lowering_prompts(&dao, dao.run.inventory());
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
panic!("answered lowering from nothing")
};
let want_lower: Vec<String> = lowering.iter().map(|(_, p)| p.digest()).collect();
assert_eq!(digests, want_lower);
assert!(dao.run.rounds().unwrap().is_empty());
assert!(!round1.join("packets.cc").exists());
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
for (_, p) in &lowering {
assert_eq!(
std::fs::read_to_string(round1.join(format!("{}.prompt", p.digest()))).unwrap(),
p.text()
);
}
// Two of three lowerings is still nothing written; the third completes round 1.
let table = table_english();
let lowered = |key: &str| serde_json::json!({ "text": table[key] }).to_string();
for (key, p) in &lowering[..2] {
std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap();
}
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let RoundOutcome::Unanswered { prompts: digests } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
panic!("answered from two of three lowerings")
};
assert_eq!(digests, [want_lower[2].clone()]);
assert!(dao.run.rounds().unwrap().is_empty());
assert!(!round1.join("packets.cc").exists());
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
let (key, p) = &lowering[2];
std::fs::write(round1.join(format!("{}.answer", p.digest())), lowered(key)).unwrap();
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!((r.n(), r.calls().len()), (1, 6));
assert_eq!(two_dp(r.language("en-US").unwrap().train_mean.chrf.unwrap()), 44.18);
// Round 2 asks again, encode first.
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
assert!(dao.root.join("round-2").parent().is_some());
let RoundOutcome::Unanswered { prompts: again } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
panic!("round 2 answered from round 1's files")
};
assert_eq!(again.len(), 3);
assert_eq!(
std::fs::read_dir(dao.root.join("round-2")).unwrap().count(),
3
);
assert_eq!(dao.run.rounds().unwrap().len(), 1);
// Answered in both phases, round 2 is recorded as round 2, after round 1.
for (key, p) in expected.iter().chain(&lowering) {
let answer = answers.get(key).cloned().unwrap_or_else(|| lowered(key));
std::fs::write(
dao.root.join(format!("round-2/{}.answer", p.digest())),
answer,
)
.unwrap();
}
let mut files = AnswerFiles::new(dao.run.next_round_dir().unwrap(), "fake-model".into());
let second = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(second.n(), 2);
let ns: Vec<u32> = dao.run.rounds().unwrap().iter().map(|r| r.n()).collect();
assert_eq!(ns, [1, 2]);
assert!(dao.root.join("round-2/packets.cc").is_file());
assert_eq!(dao.run.next_round_dir().unwrap(), dao.root.join("round-3"));
}
struct OtherEmbedder;
impl Embedder for OtherEmbedder {
fn model(&self) -> &str {
"other-embedder"
}
fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
Bag.embed(texts)
}
}
#[test]
fn drift_refuses_the_round() {
let mut dao = dao();
let mut other = fake(&good_answers());
other.name = "other-model";
assert_eq!(
round(&mut dao.run, &mut other, &Scorer::new(Bag)).err(),
Some(RunError::Drift(DriftOf::Model))
);
let mut model = fake(&good_answers());
assert_eq!(
round(&mut dao.run, &mut model, &Scorer::new(OtherEmbedder)).err(),
Some(RunError::Drift(DriftOf::Embedder))
);
assert!(
other.seen.is_empty() && model.seen.is_empty(),
"drift is found before any call"
);
assert!(dao.run.rounds().unwrap().is_empty() && !dao.root.join("round-1").exists());
// An inventory with no revision has no round.
let w = world();
std::fs::write(w.dir.path().join("i.cc"), [0x90]).unwrap();
let mut empty = run_over(w.dir.path());
let mut model = fake(&[]);
assert_eq!(
round(&mut empty, &mut model, &Scorer::new(Bag)).err(),
Some(RunError::NoInventory)
);
}
/// `n`'s good draft, edited.
fn edited(n: usize, edit: impl FnOnce(&mut PacketDraft)) -> String {
let mut d = draft_of(n);
edit(&mut d);
json(&d)
}
fn unknown_concept(d: &mut PacketDraft) {
d.referents[0].concept = "concept:999".parse().unwrap();
}
fn with_relation(d: &mut PacketDraft) {
d.relations.push(weksa::packet::Relation {
relation: "relation:1".parse().unwrap(),
from: d.referents[0].id.clone(),
to: d.referents[1].id.clone(),
features: vec![],
});
}
#[test]
fn refusal_stages_are_counted() {
let run = |answers: [String; 3]| {
let mut dao = dao();
let mut model = fake(
&answers
.into_iter()
.enumerate()
.map(|(i, a)| (i + 1, a))
.collect::<Vec<_>>(),
);
recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap())
};
let junk = "not json at all".to_string();
// Train refused at Parse; Validation refused at Admit, and a relation
// lowers (the model renders relations; the deleted table did not).
let r = run([
junk.clone(),
edited(2, unknown_concept),
edited(3, with_relation),
]);
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse));
assert_eq!(r.train()[0].packet, None);
// A refused item is in the mean as 0: n counts every item, scored the real ones.
let m = &r.language("en-US").unwrap().train_mean;
assert_eq!((m.count, m.chrf, m.meaning), (1, Some(0.0), Some(0.0)));
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (0, 1));
assert_eq!(r.language("en-US").unwrap().validation_refused.parse, 0);
assert_eq!(
(r.language("en-US").unwrap().validation_refused.admit, r.language("en-US").unwrap().validation_refused.lower),
(1, 0)
);
let m = &r.language("en-US").unwrap().validation_mean;
assert_eq!(m.count, 2);
assert!(m.chrf.unwrap() > 0.0, "the relation item lowered and scored");
// Validation items 2 and 3 score a and b when both lower (the Dao bed:
// 82.12 and 74.99). Refusing one halves the mean; it cannot raise it.
let full = run([
good_answers()[0].1.clone(),
good_answers()[1].1.clone(),
good_answers()[2].1.clone(),
]);
let b_only = run([
edited(1, unknown_concept),
junk.clone(),
good_answers()[2].1.clone(),
]);
assert_eq!(b_only.train()[0].outcome, Outcome::Refused(Stage::Admit));
assert_eq!(
b_only.train()[0].packet,
None,
"a refused packet is not stored"
);
let v = b_only.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0));
assert_eq!(
b_only.language("en-US").unwrap().validation_mean.count,
2,
"every Validation item counts"
);
assert_eq!(b_only.language("en-US").unwrap().validation_scored, 1);
let b_half = b_only.language("en-US").unwrap().validation_mean.chrf.unwrap();
assert!(
(b_half - 74.99 / 2.0).abs() < 0.01,
"(74.99 + 0) / 2 = {b_half}"
);
let a_only = run([good_answers()[0].1.clone(), good_answers()[1].1.clone(), junk]);
assert_eq!(a_only.language("en-US").unwrap().validation_refused.parse, 1);
let a_half = a_only.language("en-US").unwrap().validation_mean.chrf.unwrap();
assert!(
(a_half - 82.12 / 2.0).abs() < 0.01,
"(82.12 + 0) / 2 = {a_half}"
);
// The halves add to the full mean: (a + 0) / 2 + (0 + b) / 2 = (a + b) / 2.
let whole = full.language("en-US").unwrap().validation_mean.chrf.unwrap();
assert!(
(a_half + b_half - whole).abs() < 1e-9,
"{a_half} + {b_half} vs {whole}"
);
assert!(
b_half < whole && a_half < whole,
"refusing never raises the mean"
);
assert_eq!(full.language("en-US").unwrap().validation_scored, 2);
}
#[test]
fn non_utf8_encode_answer_is_a_parse_refusal() {
let mut dao = dao();
let expected = prompts(&dao.run);
let answers: HashMap<String, String> = good_answers()
.into_iter()
.map(|(n, a)| (format!("item:daodejing:{n}"), a))
.collect();
let round1 = dao.root.join("round-1");
std::fs::create_dir_all(&round1).unwrap();
for (key, p) in &expected {
let bytes = if key == "item:daodejing:3" {
vec![0xff, 0xfe, 0xfd]
} else {
answers[key].clone().into_bytes()
};
std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap();
}
// Item 3 never reaches the second phase.
let table = table_english();
for (key, p) in lowering_prompts(&dao, dao.run.inventory())
.iter()
.filter(|(key, _)| key != "item:daodejing:3")
{
let answer = serde_json::json!({ "text": table[key] }).to_string();
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1.clone(), "fake-model".into());
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(r.calls().len(), 5);
// Item 3 is Validation: its refusal is a Parse refusal and the round stands.
let v = r.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0));
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 1));
assert_eq!(r.language("en-US").unwrap().validation_mean.count, 2);
assert_eq!(dao.run.rounds().unwrap(), [r]);
}
/// An embedder that is down.
struct Down;
impl Embedder for Down {
fn model(&self) -> &str {
"bag"
}
fn embed(&self, _: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
anyhow::bail!("down")
}
}
#[test]
fn a_failed_round_leaves_nothing_a_rerun_collides_with() {
let mut dao = dao();
let mut model = fake(&good_answers());
assert_eq!(
round(&mut dao.run, &mut model, &Scorer::new(Down)).err(),
Some(RunError::Embed)
);
assert!(dao.run.rounds().unwrap().is_empty());
assert!(
!dao.root.join("round-1/packets.cc").exists(),
"the failed attempt's packets are discarded"
);
// The session revises item 1's answer and reruns: its packet is admitted
// as the revised one, not refused as a changed packet under the same key.
let mut revised = draft_of(1);
revised.confidence = weksa::packet::Grade::new(123).unwrap();
let mut answers = good_answers();
answers[0].1 = json(&revised);
let mut model = fake(&answers);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert!(
matches!(r.train()[0].outcome, Outcome::Scored { .. }),
"{:?}",
r.train()[0]
);
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
let stored = PacketStore::open(&dao.root.join("round-1/packets.cc"))
.unwrap()
.all()
.unwrap();
assert_eq!(stored.len(), 3);
assert!(stored.iter().any(|p| p.provenance.confidence.get() == 123));
}
#[test]
fn a_stale_packet_store_in_an_unrecorded_round_dir_is_discarded_at_start() {
// A previous attempt that died after admitting (a crash, a kill) left its
// packets and lock in the next round's directory.
let mut first = dao();
let mut model = fake(&good_answers());
recorded(round(&mut first.run, &mut model, &Scorer::new(Bag)).unwrap());
let mut dao = dao();
let dir = dao.root.join("round-1");
std::fs::create_dir_all(&dir).unwrap();
std::fs::copy(
first.root.join("round-1/packets.cc"),
dir.join("packets.cc"),
)
.unwrap();
std::fs::write(dir.join("packets.cc.lock"), "stale").unwrap();
// The session revised item 1's answer; the stale packet for it is the old one.
let mut revised = draft_of(1);
revised.confidence = weksa::packet::Grade::new(321).unwrap();
let mut answers = good_answers();
answers[0].1 = json(&revised);
let mut model = fake(&answers);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert!(
matches!(r.train()[0].outcome, Outcome::Scored { .. }),
"the revised draft is not refused as a changed packet: {:?}",
r.train()[0]
);
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
let stored = PacketStore::open(&dir.join("packets.cc"))
.unwrap()
.all()
.unwrap();
assert_eq!(stored.len(), 3);
assert!(stored.iter().any(|p| p.provenance.confidence.get() == 321));
assert!(
!stored.iter().any(|p| p.provenance.confidence.get() == 600
&& p.provenance.corpus_item.as_str() == "item:daodejing:1"),
"the stale packet is gone"
);
assert_ne!(
std::fs::read(dir.join("packets.cc.lock")).unwrap_or_default(),
b"stale",
"the stale lock was removed"
);
}
#[test]
fn the_round_line_prints_each_split_under_its_own_name() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let line = dao.run.round_line(&r);
// Train is item 1 alone, Validation items 2 and 3; the numbers differ.
let (t, v) = (&r.language("en-US").unwrap().train_mean, &r.language("en-US").unwrap().validation_mean);
assert_eq!(
line,
format!(
"round 1 rev 3 en-US train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0 lowering=tool-less-relay",
t.chrf.unwrap(),
t.meaning.unwrap(),
v.chrf.unwrap(),
v.meaning.unwrap()
)
);
assert!(
line.contains("train n=1 scored=1 chrf=44.18 meaning=0."),
"{line}"
);
assert!(
line.contains("validation n=2 scored=2 chrf=78.56 meaning=0."),
"{line}"
);
assert_ne!(t.meaning, v.meaning);
}
#[test]
fn errors_echo_no_input() {
let canary = "CANARYBYTES";
let tmp = tempfile::tempdir().unwrap();
let mut all: Vec<String> = Vec::new();
let mut note = |e: &dyn std::fmt::Debug, d: &dyn std::fmt::Display| {
all.push(format!("{e:?}"));
all.push(d.to_string());
};
// A draft error, from canary bytes in an answer.
let e = PacketDraft::parse(&format!("{{\"confidence\": {canary}}}")).unwrap_err();
note(&e, &e);
let e = PacketDraft::parse(&format!("{{\"{canary}\": 1}}")).unwrap_err();
note(&e, &e);
// Run errors, from a canary root, label and store path.
let root = tmp.path().join("canaryroot");
let e = RunStores::open(&root).err().unwrap();
note(&e, &e);
let missing = tmp.path().join("canarystore.cc");
let stores = RunPaths {
corpus: &missing,
inventory: &missing,
entities: &missing,
};
let e = RunStores::create(
&root,
"ok",
stores,
"m",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.err()
.unwrap();
note(&e, &e);
let e = RunStores::create(
&root,
&format!("{canary}!"),
stores,
"m",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.err()
.unwrap();
note(&e, &e);
let e = RunStores::create(
&root,
"ok",
stores,
&format!("{canary}\n"),
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.err()
.unwrap();
note(&e, &e);
let w = world();
let run = run_over(w.dir.path());
let e = run.offer(&format!("item:{canary}:1")).err().unwrap();
note(&e, &e);
// A port error, from a canary directory.
let blocker = tmp.path().join("canaryfile");
std::fs::write(&blocker, "").unwrap();
let mut files = AnswerFiles::new(blocker.join("canarydir"), "m".into());
let e = files.answer(&prompt()).err().unwrap();
note(&e, &e);
// A round over canary answers records refusals, not the answers.
let mut dao = dao();
let mut model = fake(&[
(1, canary.into()),
(2, format!("{{\"{canary}\":1}}")),
(3, canary.into()),
]);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
all.push(format!("{r:?}"));
// Cycle errors, from a canary root, and a decision over a canary edit answer.
let canary_root = tmp.path().join("canaryrun");
let mut canary_run = RunStores::create(
&canary_root,
"ok",
RunPaths {
corpus: &dao.corpus,
inventory: &dao.inventory,
entities: &dao.entities,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
let mut model = cycle_fake(&format!("{{\"{canary}\":1}}"));
let e = cycle(&mut canary_run, &mut model, &Scorer::new(Bag))
.err()
.unwrap();
assert_eq!(e, RunError::NoFloor);
all.extend([format!("{e:?}"), e.to_string()]);
for _ in 0..2 {
let mut m = fake(&degraded_answers());
recorded(round(&mut canary_run, &mut m, &Scorer::new(Bag)).unwrap());
}
let d = decided(cycle(&mut canary_run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(d.verdict(), Verdict::Rejected(Why::EditParse));
all.extend([format!("{d:?}"), canary_run.cycle_line(&d)]);
let mut non_utf8 = cycle_fake("");
non_utf8.edit = Some(Err(PortError::Io(std::io::ErrorKind::InvalidData)));
let d = decided(cycle(&mut canary_run, &mut non_utf8, &Scorer::new(Bag)).unwrap());
all.extend([format!("{d:?}"), canary_run.cycle_line(&d)]);
weksa::InventoryStore::open(&canary_root.join("inventory.cc"))
.unwrap()
.admit_revision(
weksa::InventoryRevision {
n: 4,
parent: Some(3),
changes: vec![],
frozen: false,
},
vec![],
)
.unwrap();
let e = RunStores::open(&canary_root).err().unwrap();
assert_eq!(e, RunError::Drift(DriftOf::Inventory));
all.extend([format!("{e:?}"), e.to_string()]);
let e = RunError::NoValidation;
all.extend([format!("{e:?}"), e.to_string()]);
assert_eq!(
format!("{:?}", RunError::NoFloor),
RunError::NoFloor.to_string()
);
for text in &all {
assert!(!text.to_uppercase().contains("CANARY"), "{text}");
}
assert!(all.len() >= 22);
// Debug names the same kind Display does.
assert_eq!(format!("{:?}", RunError::NoRun), "the root holds no run");
assert_eq!(
format!("{:?}", RunError::Drift(DriftOf::Inventory)),
"drift in Inventory"
);
assert_eq!(
format!("{:?}", RunError::Drift(DriftOf::Model)),
"drift in Model"
);
// The Prompts text names the three files its digest covers.
let prompts = RunError::Drift(DriftOf::Prompts).to_string();
assert!(["context.rs", "split.rs", "lower.rs", "packet.rs"].iter().all(|f| prompts.contains(f)));
assert_eq!(format!("{:?}", RunError::Drift(DriftOf::Prompts)), prompts);
}
#[test]
fn prompts_come_only_from_context() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(model.seen.len(), r.calls().len());
assert_eq!(model.seen.len(), 6);
for ((digest, text), call) in model.seen.iter().zip(r.calls()) {
assert_eq!(call.items.len(), 1);
if text.starts_with(ENCODE_INSTRUCTIONS) {
assert_eq!(text.matches("[item:").count(), 1);
assert!(text.contains(&format!("[{}]", call.items[0])));
} else {
// A lowering prompt holds no item block and prints no item key.
assert!(text.starts_with(LOWER_INSTRUCTIONS));
assert_eq!(text.matches("[item:").count(), 0);
assert!(!text.contains("item:daodejing"));
}
assert_eq!(&call.prompt, digest);
assert_eq!(*digest, format!("{:016x}", fnv1a64(text.as_bytes())));
}
}
// ---- edit-accept cycles ----
use weksa::run::{CycleOutcome, Decision, LADDER_STEP, Reading, Verdict, Why, cycle};
const ONE_CONCEPT: &str =
r#"{"add":[{"family":"Concept","gloss":"a general meaning the Dao names"}]}"#;
/// Item 2 refused at Admit (an unknown concept), items 1 and 3 good.
fn degraded_answers() -> Vec<(usize, String)> {
vec![
(1, json(&draft_of(1))),
(2, edited(2, unknown_concept)),
(3, json(&draft_of(3))),
]
}
/// Head answers degrade item 2; candidate answers (for `inventory rev:4`) are
/// the good Dao drafts; the editor answers `edit`.
fn cycle_fake(edit: &str) -> FakeModel {
let mut model = fake(&degraded_answers());
model.candidate = Some((4, keyed(&good_answers())));
model.edit = Some(Ok(edit.to_string()));
model
}
fn two_rounds(dao: &mut Dao) {
for _ in 0..2 {
let mut model = fake(&degraded_answers());
recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
}
}
fn decided(outcome: CycleOutcome) -> Decision {
match outcome {
CycleOutcome::Decided(d) => d,
CycleOutcome::Unanswered { prompts } => panic!("unanswered: {prompts:?}"),
}
}
fn run_cycle(dao: &mut Dao, model: &mut FakeModel) -> Decision {
decided(cycle(&mut dao.run, model, &Scorer::new(Bag)).unwrap())
}
fn head_rev(root: &Path) -> u32 {
let run = RunStores::open(root).unwrap();
let inv = run.inventory().inventory().unwrap();
inv.latest().unwrap().n
}
fn validation(r: &RoundRecord) -> Reading {
Reading::of(&r.language("en-US").unwrap().validation_mean).unwrap()
}
/// The feedback line the editor is shown for a head round, built from outside.
fn feedback_line(last: &str, head: &RoundRecord) -> String {
let rounded = (validation(head).composite / LADDER_STEP).round() * LADDER_STEP;
format!("last edit: {last}; validation composite at head: en-US {rounded:.3}")
}
fn mint_texts(model: &FakeModel) -> Vec<&str> {
model
.seen
.iter()
.map(|(_, t)| t.as_str())
.filter(|t| t.starts_with(MINT_INSTRUCTIONS))
.collect()
}
#[test]
fn cycle_accepts_a_gain_past_the_floor() {
let mut dao = dao();
two_rounds(&mut dao);
let source = std::fs::read(&dao.inventory).unwrap();
let rounds = dao.run.rounds().unwrap();
assert_eq!(validation(&rounds[0]), validation(&rounds[1]), "floor 0");
let mut model = cycle_fake(ONE_CONCEPT);
let d = run_cycle(&mut dao, &mut model);
assert_eq!(
(d.k(), d.base_round(), d.candidate_round()),
(1, 1, Some(3))
);
assert_eq!((d.base_rev(), d.candidate_rev()), (3, Some(4)));
assert_eq!(d.verdict(), Verdict::Accepted);
assert_eq!(d.panel()[0].threshold, LADDER_STEP);
assert_eq!(
d.panel()[0].floor,
Reading {
chrf: 0.0,
meaning: 0.0,
composite: 0.0
}
);
assert_eq!(d.panel()[0].base, validation(&rounds[0]));
let record = dao.run.rounds().unwrap().remove(2);
assert_eq!(d.panel()[0].candidate, Some(validation(&record)));
assert!(d.panel()[0].candidate.unwrap().composite - d.panel()[0].base.composite > LADDER_STEP);
// The candidate round was scored under the candidate file, as cycle 1's.
let file = dao.root.join("cycle-1/inventory.cc");
assert_eq!(
(record.n(), record.inventory_rev(), record.cycle()),
(3, 4, Some(1))
);
assert_eq!(record.inventory_digest(), hex(&file));
assert_eq!(d.candidate_inventory_digest(), Some(hex(&file).as_str()));
assert_eq!(
d.base_inventory_digest(),
hex(&dao.root.join("inventory.cc"))
);
assert_eq!(d.base_inventory_digest(), dao.run.record().inventory_digest);
assert!(dao.root.join("cycle-1/packets.cc").is_file());
assert_eq!(d.edit().items, ["item:daodejing:1"]);
assert_eq!(
d.edit_answer_digest(),
Some(format!("{:016x}", fnv1a64(ONE_CONCEPT.as_bytes())).as_str())
);
assert_eq!(d.edit().prompt, model.seen[0].0);
// The line, with the readings printed to four places.
let (b, c) = (d.panel()[0].base.composite, d.panel()[0].candidate.unwrap().composite);
assert_eq!(
dao.run.cycle_line(&d),
format!(
"cycle 1 rev 3->4 en-US base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted lowering=tool-less-relay"
)
);
// The decision made it the head: a reopened run reads rev 4 from the cycle's
// file, and the caller's inventory file was never touched.
assert_eq!(head_rev(&dao.root), 4);
assert_eq!(std::fs::read(&dao.inventory).unwrap(), source);
assert_eq!(dao.run.decisions().unwrap(), std::slice::from_ref(&d));
assert_eq!(
RunStores::open(&dao.root).unwrap().decisions().unwrap(),
std::slice::from_ref(&d)
);
for (_, p) in prompts(&dao.run) {
assert!(p.text().contains("\n\ninventory rev:4\n"));
}
// `loop round` reads the head and never moves it; the next cycle's base is
// the establishing round 3, not the replicate recorded after it.
let mut again = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut again, &Scorer::new(Bag)).unwrap());
assert_eq!((r.n(), r.inventory_rev(), r.cycle()), (4, 4, None));
assert_eq!(r.inventory_digest(), hex(&file));
let mut second = fake(&good_answers());
second.candidate = Some((5, keyed(&good_answers())));
second.edit = Some(Ok(ONE_CONCEPT.to_string()));
let d2 = run_cycle(&mut dao, &mut second);
assert_eq!((d2.k(), d2.base_round(), d2.base_rev()), (2, 3, 4));
assert_eq!(d2.candidate_rev(), Some(5));
assert_eq!(d2.candidate_round(), Some(5));
assert_eq!(d2.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!(d2.panel()[0].floor, d.panel()[0].floor, "the floor is measured once");
assert_eq!(head_rev(&dao.root), 4, "a rejected edit moves nothing");
assert!(
mint_texts(&second)[0].contains(&feedback_line("accepted", &dao.run.rounds().unwrap()[2])),
"{}",
mint_texts(&second)[0]
);
assert!(dao.root.join("cycle-2/inventory.cc").is_file());
assert_eq!(
d2.base_inventory_digest(),
d.candidate_inventory_digest().unwrap()
);
}
#[test]
fn cycle_rejects_no_gain_and_keeps_the_head() {
let mut dao = dao();
two_rounds(&mut dao);
let mut model = fake(&degraded_answers());
model.candidate = Some((4, keyed(&degraded_answers())));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
let d = run_cycle(&mut dao, &mut model);
assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!((d.candidate_rev(), d.candidate_round()), (Some(4), Some(3)));
assert_eq!(d.panel()[0].candidate, Some(d.panel()[0].base));
let c = d.panel()[0].base.composite;
assert_eq!(
dao.run.cycle_line(&d),
format!(
"cycle 1 rev 3->4 en-US base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain lowering=tool-less-relay"
)
);
assert_eq!(head_rev(&dao.root), 3);
assert_eq!(
RunStores::open(&dao.root)
.unwrap()
.record()
.inventory_digest,
hex(&dao.root.join("inventory.cc"))
);
assert_eq!(
dao.run.rounds().unwrap().len(),
3,
"the candidate round is recorded"
);
// Cycle 2 goes to its own directory and is told the last edit was rejected,
// with the same head composite.
assert_eq!(dao.run.next_cycle_dir().unwrap(), dao.root.join("cycle-2"));
let d2 = run_cycle(&mut dao, &mut model);
assert_eq!((d2.k(), d2.base_round(), d2.base_rev()), (2, 1, 3));
assert!(dao.root.join("cycle-2/inventory.cc").is_file());
let mints = mint_texts(&model);
assert_eq!(mints.len(), 2);
let round1 = dao.run.rounds().unwrap().remove(0);
assert!(mints[0].contains(&feedback_line("none", &round1)));
assert!(mints[1].contains(&feedback_line("rejected", &round1)));
}
/// Item 2 and item 3 both refused at Admit: the weakest reading.
fn worst_answers() -> Vec<(usize, String)> {
vec![
(1, json(&draft_of(1))),
(2, edited(2, unknown_concept)),
(3, edited(3, unknown_concept)),
]
}
#[test]
fn distinct_replicates_set_the_floor_and_a_gain_inside_it_is_no_gain() {
let mut dao = dao();
// Replicate 1 is degraded, replicate 2 is good: the two readings differ.
for answers in [degraded_answers(), good_answers()] {
let mut m = fake(&answers);
recorded(round(&mut dao.run, &mut m, &Scorer::new(Bag)).unwrap());
}
let rounds = dao.run.rounds().unwrap();
let (a, b) = (validation(&rounds[0]), validation(&rounds[1]));
let want = Reading {
chrf: (a.chrf - b.chrf).abs(),
meaning: (a.meaning - b.meaning).abs(),
composite: (a.composite - b.composite).abs(),
};
assert!(want.chrf > 0.0 && want.meaning > 0.0 && want.composite > LADDER_STEP);
// The candidate reads exactly as the good replicate: its gain over the head
// (round 1) is the floor, which is past a step and inside the noise.
let mut model = cycle_fake(ONE_CONCEPT);
let d = run_cycle(&mut dao, &mut model);
assert_eq!(d.panel()[0].floor, want);
assert_eq!(d.panel()[0].threshold, want.composite);
assert_eq!(d.panel()[0].candidate, Some(b));
assert!(d.panel()[0].candidate.unwrap().composite - d.panel()[0].base.composite > LADDER_STEP);
assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!(head_rev(&dao.root), 3);
}
#[test]
fn a_rejected_candidates_score_never_reaches_the_editor() {
let mut dao = dao();
two_rounds(&mut dao);
let mut model = fake(&degraded_answers());
model.candidate = Some((4, keyed(&worst_answers())));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
let d = run_cycle(&mut dao, &mut model);
assert!(matches!(d.verdict(), Verdict::Rejected(Why::Regressed(_))));
let rounds = dao.run.rounds().unwrap();
let (head, candidate) = (&rounds[0], &rounds[2]);
let ladder = |r: &RoundRecord| feedback_line("rejected", r);
assert_ne!(
ladder(head),
ladder(candidate),
"the candidate reads differently on the ladder"
);
let d2 = run_cycle(&mut dao, &mut model);
assert_eq!(d2.base_round(), 1);
let mints = mint_texts(&model);
assert!(mints[1].contains(&ladder(head)), "{}", mints[1]);
assert!(!mints[1].contains(&ladder(candidate)), "{}", mints[1]);
let rounds = dao.run.rounds().unwrap();
assert!(!mints[1].contains(&ladder(&rounds[3])), "{}", mints[1]);
}
#[test]
fn two_accepts_make_the_latest_the_head_and_a_reject_after_them_is_told_so() {
let mut dao = dao();
// Level 0 twice: a floor of 0 and a base reading of the weakest.
for _ in 0..2 {
let mut m = fake(&worst_answers());
recorded(round(&mut dao.run, &mut m, &Scorer::new(Bag)).unwrap());
}
let mut model = fake(&worst_answers());
let mut step = |dao: &mut Dao, rev: u32, answers: &[(usize, String)]| {
model.candidate = Some((rev, keyed(answers)));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
run_cycle(dao, &mut model)
};
let d1 = step(&mut dao, 4, &degraded_answers());
let d2 = step(&mut dao, 5, &good_answers());
assert_eq!(d1.verdict(), Verdict::Accepted);
assert_eq!(d2.verdict(), Verdict::Accepted);
assert_eq!((d2.base_round(), d2.base_rev()), (3, 4));
assert_eq!(
(d2.candidate_round(), d2.candidate_rev()),
(Some(4), Some(5))
);
// The head is the second accept, live and reopened.
assert_eq!(head_rev(&dao.root), 5);
assert_eq!(
d2.candidate_inventory_digest(),
Some(hex(&dao.root.join("cycle-2/inventory.cc")).as_str())
);
// Cycle 3 builds on cycle 2's file and round, gains nothing, and is rejected.
let d3 = step(&mut dao, 6, &good_answers());
assert_eq!((d3.k(), d3.base_round(), d3.base_rev()), (3, 4, 5));
assert_eq!(
d3.base_inventory_digest(),
d2.candidate_inventory_digest().unwrap()
);
assert_eq!(d3.verdict(), Verdict::Rejected(Why::NoGain));
assert_eq!(head_rev(&dao.root), 5, "a reject moves nothing");
// Cycle 4 is told by the latest decision alone: cycle 3's reject, not cycle
// 2's accept, and the composite at the head, which the reject did not move.
let d4 = step(&mut dao, 6, &good_answers());
assert_eq!((d4.k(), d4.base_round(), d4.base_rev()), (4, 4, 5));
let rounds = dao.run.rounds().unwrap();
let mints = mint_texts(&model);
assert_eq!(mints.len(), 4);
assert!(mints[0].contains(&feedback_line("none", &rounds[0])));
assert!(mints[1].contains(&feedback_line("accepted", &rounds[2])));
assert!(mints[2].contains(&feedback_line("accepted", &rounds[3])));
assert!(
mints[3].contains(&feedback_line("rejected", &rounds[3])),
"{}",
mints[3]
);
assert!(mints[1].contains("\n\ninventory rev:4\n"));
assert!(mints[3].contains("\n\ninventory rev:5\n"));
}
#[test]
fn a_rejection_leaves_no_candidate_inventory_from_an_earlier_attempt() {
for (name, rewrite, why) in [
("zero changes", "{}", Why::EditAdmit),
("not json", "not json", Why::EditParse),
] {
let mut dao = dao();
two_rounds(&mut dao);
let scorer = Scorer::new(Bag);
let ask = |dao: &mut Dao| {
let mut files =
AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into());
cycle(&mut dao.run, &mut files, &scorer).unwrap()
};
let dir = dao.root.join("cycle-1");
let CycleOutcome::Unanswered { prompts } = ask(&mut dao) else {
panic!("{name}: answered from nothing")
};
let answer = dir.join(format!("{}.answer", prompts[0]));
std::fs::write(&answer, ONE_CONCEPT).unwrap();
// The candidate round is unanswered, which leaves the candidate inventory.
let CycleOutcome::Unanswered { .. } = ask(&mut dao) else {
panic!("{name}: answered the candidate round from nothing")
};
assert!(dir.join("inventory.cc").is_file(), "{name}");
// The editor's answer is replaced by one the door refuses.
std::fs::write(&answer, rewrite).unwrap();
let d = decided(ask(&mut dao));
assert_eq!(d.verdict(), Verdict::Rejected(why), "{name}");
assert!(!dir.join("inventory.cc").exists(), "{name}");
assert!(!dir.join("inventory.cc.lock").exists(), "{name}");
assert_eq!(head_rev(&dao.root), 3, "{name}");
}
}
#[cfg(unix)]
#[test]
fn a_failed_accept_write_leaves_neither_the_round_nor_the_decision() {
use std::os::unix::fs::PermissionsExt;
let mut dao = dao();
two_rounds(&mut dao);
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
let mut model = cycle_fake(ONE_CONCEPT);
// The cycle's directory exists, so everything but run.cc can still be
// written once the run's root takes no new files: the batch's staging file
// is refused, and it is the only write to run.cc a cycle makes.
std::fs::create_dir_all(dao.root.join("cycle-1")).unwrap();
let chmod = |mode| std::fs::set_permissions(&dao.root, std::fs::Permissions::from_mode(mode));
chmod(0o500).unwrap();
if std::fs::File::create(dao.root.join("probe")).is_ok() {
// Running as a user the mode does not bind: the fault cannot be injected.
std::fs::remove_file(dao.root.join("probe")).unwrap();
chmod(0o700).unwrap();
return;
}
let err = cycle(&mut dao.run, &mut model, &Scorer::new(Bag)).err();
chmod(0o700).unwrap();
assert_eq!(err, Some(RunError::Store));
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
assert_eq!(dao.run.rounds().unwrap().len(), 2, "no orphan round");
assert!(dao.run.decisions().unwrap().is_empty(), "no decision");
let reopened = RunStores::open(&dao.root).unwrap();
assert_eq!(reopened.rounds().unwrap().len(), 2);
assert!(reopened.decisions().unwrap().is_empty());
assert_eq!(head_rev(&dao.root), 3);
// The rerun decides cycle 1 on round 3, as if the failure had not happened.
let d = run_cycle(&mut dao, &mut model);
assert_eq!((d.k(), d.candidate_round()), (1, Some(3)));
assert_eq!(d.verdict(), Verdict::Accepted);
}
#[test]
fn unanswered_cycle_writes_nothing() {
let mut dao = dao();
two_rounds(&mut dao);
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
let scorer = Scorer::new(Bag);
let ask = |dao: &mut Dao| {
let mut files = AnswerFiles::new(dao.run.next_cycle_dir().unwrap(), "fake-model".into());
cycle(&mut dao.run, &mut files, &scorer).unwrap()
};
let dir = dao.root.join("cycle-1");
// The first call asks the editor and nothing else.
let CycleOutcome::Unanswered { prompts: asked } = ask(&mut dao) else {
panic!("answered from nothing")
};
assert_eq!(asked.len(), 1);
let mint_file = dir.join(format!("{}.prompt", asked[0]));
let mint = std::fs::read_to_string(&mint_file).unwrap();
assert_eq!(format!("{:016x}", fnv1a64(mint.as_bytes())), asked[0]);
assert!(mint.starts_with(MINT_INSTRUCTIONS));
assert!(mint.contains("\n\nlast edit: none; validation composite at head: en-US 0.435\n\n"));
assert!(mint.contains("\n\ninventory rev:3\n"));
assert_eq!(mint.matches("[item:").count(), 1, "one Train block");
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1);
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
// With the edit answered, the candidate round asks three encode prompts at rev 4.
std::fs::write(dir.join(format!("{}.answer", asked[0])), ONE_CONCEPT).unwrap();
let CycleOutcome::Unanswered { prompts: encode } = ask(&mut dao) else {
panic!("answered the candidate round from nothing")
};
assert_eq!(encode.len(), 3);
let answers: HashMap<String, String> = keyed(&good_answers());
for digest in &encode {
let text = std::fs::read_to_string(dir.join(format!("{digest}.prompt"))).unwrap();
assert!(text.contains("\n\ninventory rev:4\n"), "{text}");
assert!(text.starts_with(ENCODE_INSTRUCTIONS));
let key = text
.split("[item:")
.nth(1)
.and_then(|t| t.split(']').next())
.map(|k| format!("item:{k}"))
.unwrap();
std::fs::write(dir.join(format!("{digest}.answer")), &answers[&key]).unwrap();
}
assert_eq!(
std::fs::read(dao.root.join("run.cc")).unwrap(),
run_cc,
"no round 3 and no decision"
);
assert_eq!(dao.run.rounds().unwrap().len(), 2);
assert!(dao.run.decisions().unwrap().is_empty());
// With the encodes answered, the candidate round asks its lowerings, listing
// the candidate inventory; still nothing is written.
let candidate = InventoryStore::open(&dir.join("inventory.cc")).unwrap();
let lowering = lowering_prompts(&dao, &candidate);
let CycleOutcome::Unanswered { prompts: lower } = ask(&mut dao) else {
panic!("answered the candidate lowerings from nothing")
};
assert_eq!(
lower,
lowering.iter().map(|(_, p)| p.digest()).collect::<Vec<_>>()
);
assert!(lowering[0].1.text().contains("\n\ninventory rev:4\n"));
assert!(!dir.join("packets.cc").exists());
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
assert_eq!(dao.run.rounds().unwrap().len(), 2);
assert!(dao.run.decisions().unwrap().is_empty());
let table = table_english();
for (key, p) in &lowering {
let answer = serde_json::json!({ "text": table[key] }).to_string();
std::fs::write(dir.join(format!("{}.answer", p.digest())), answer).unwrap();
}
// Answered, the cycle is decided; the next one uses cycle-2.
let d = decided(ask(&mut dao));
assert_eq!((d.k(), d.verdict()), (1, Verdict::Accepted));
assert_eq!(dao.run.next_cycle_dir().unwrap(), dao.root.join("cycle-2"));
let CycleOutcome::Unanswered { prompts: next } = ask(&mut dao) else {
panic!("cycle 2 answered from cycle 1's files")
};
assert_eq!(next.len(), 1);
assert!(
dao.root
.join(format!("cycle-2/{}.prompt", next[0]))
.is_file()
);
assert_ne!(next[0], asked[0]);
}
#[test]
fn the_editor_sees_train_only() {
run_the_editor_canary(CANARY_LOWERED);
}
/// What the lowerer says for the canary Validation item.
const CANARY_LOWERED: &str = "CANARYLOWERED";
#[test]
fn validation_lowerings_never_reach_the_editor() {
// The same run under another lowering for the canary: the editor's prompts
// never print either, and nothing recorded holds either.
run_the_editor_canary("CANARYLOWERED");
run_the_editor_canary("the zebra ate the quartz");
}
fn run_the_editor_canary(canary_lowered: &str) {
let canary_key = "item:canary:1";
let canary_text = "CANARYVALIDATIONTEXT";
let mut dao = dao_over(|corpus, entities| {
let entities = weksa::EntityStore::open(entities).unwrap();
let mut store = weksa::CorpusStore::open(corpus).unwrap();
let mut item = store.get("item:daodejing:2").unwrap().unwrap();
item.id = canary_key.into();
item.text = canary_text.into();
item.names = vec![];
item.prior_turns = vec![];
store.add(item, &entities).unwrap();
});
let with_canary = |mut m: FakeModel| {
m.answers.insert(canary_key.into(), json(&draft_of(2)));
m.lowered.insert(canary_key.into(), canary_lowered.into());
if let Some((_, c)) = m.candidate.as_mut() {
c.insert(canary_key.into(), json(&draft_of(2)));
}
m
};
for _ in 0..2 {
let mut m = with_canary(fake(&degraded_answers()));
recorded(round(&mut dao.run, &mut m, &Scorer::new(Bag)).unwrap());
}
let mut model = fake(&degraded_answers());
model.candidate = Some((4, keyed(&degraded_answers())));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
let mut model = with_canary(model);
let d = run_cycle(&mut dao, &mut model);
assert_eq!(d.verdict(), Verdict::Rejected(Why::NoGain));
// A plain round with different answers changes the latest reading, not the
// head's: the line the next cycle shows is the head round's.
let mut other = with_canary(fake(&good_answers()));
let r4 = recorded(round(&mut dao.run, &mut other, &Scorer::new(Bag)).unwrap());
let rounds = dao.run.rounds().unwrap();
assert_ne!(validation(&r4), validation(&rounds[0]));
let d2 = run_cycle(&mut dao, &mut model);
assert_eq!(d2.base_round(), 1);
let corpus = weksa::CorpusStore::open(&dao.corpus).unwrap();
let mut held_out: Vec<String> = vec![
canary_key.into(),
canary_text.into(),
"canary".into(),
canary_lowered.into(),
];
for n in 2..=5 {
let item = corpus.get(&format!("item:daodejing:{n}")).unwrap().unwrap();
held_out.extend([item.id, item.text]);
}
let mints = mint_texts(&model);
assert_eq!(mints.len(), 2);
for text in &mints {
assert!(text.starts_with(MINT_INSTRUCTIONS));
for needle in &held_out {
assert!(
!text.to_lowercase().contains(&needle.to_lowercase()),
"{needle}"
);
}
assert_eq!(text.matches("[item:").count(), 1);
assert!(text.contains("[item:daodejing:1]"));
}
assert_eq!(
model.mint_keys,
[["item:daodejing:1"], ["item:daodejing:1"]]
);
// The one number from the held-out split is the head composite on the ladder.
let head = &rounds[0];
assert!(mints[0].contains(&format!("\n\n{}\n\n", feedback_line("none", head))));
assert!(mints[1].contains(&format!("\n\n{}\n\n", feedback_line("rejected", head))));
assert_eq!(
mints[0].replace("last edit: none;", "last edit: rejected;"),
mints[1],
"the two prompts differ by the last-edit word alone"
);
// Nothing the cycle writes or prints holds a held-out key or text.
for text in [
format!("{d:?}"),
dao.run.cycle_line(&d),
format!("{d2:?}"),
dao.run.cycle_line(&d2),
] {
for needle in &held_out {
assert!(
!text.to_lowercase().contains(&needle.to_lowercase()),
"{needle}: {text}"
);
}
}
// Round records list the keys of the items they called, never lowered text.
for text in [
format!("{:?}", dao.run.rounds().unwrap()),
format!("{:?}", dao.run.decisions().unwrap()),
] {
for needle in [canary_lowered, canary_text] {
assert!(!text.to_lowercase().contains(&needle.to_lowercase()));
}
}
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
for needle in [
canary_text,
canary_lowered,
"item:daodejing:4",
"item:daodejing:5",
] {
assert!(
!run_cc.windows(needle.len()).any(|w| w == needle.as_bytes()),
"{needle}"
);
}
}
#[test]
fn edit_refusals_are_decisions() {
use std::io::ErrorKind;
let nine = format!(
r#"{{"add":[{}]}}"#,
[r#"{"family":"Concept","gloss":"g"}"#; 9].join(",")
);
let long = format!(
r#"{{"add":[{{"family":"Concept","gloss":"{}"}}]}}"#,
"x".repeat(121)
);
let cases: Vec<(&str, Result<String, PortError>, Why)> = vec![
("not json", Ok("not json at all".into()), Why::EditParse),
(
"non-utf8",
Err(PortError::Io(ErrorKind::InvalidData)),
Why::EditParse,
),
(
"an id in an entry",
Ok(r#"{"add":[{"family":"Concept","gloss":"g","id":"concept:1"}]}"#.into()),
Why::EditParse,
),
("no changes", Ok("{}".into()), Why::EditAdmit),
("nine changes", Ok(nine), Why::EditAdmit),
(
"retiring an absent id",
Ok(r#"{"retire":["concept:9999"]}"#.into()),
Why::EditAdmit,
),
("a 121-byte gloss", Ok(long), Why::EditAdmit),
(
"a control character in a gloss",
Ok("{\"add\":[{\"family\":\"Concept\",\"gloss\":\"a\\u0007b\"}]}".into()),
Why::EditAdmit,
),
];
for (name, edit, why) in cases {
let mut dao = dao();
two_rounds(&mut dao);
let mut model = cycle_fake("");
model.edit = Some(edit.clone());
let d = run_cycle(&mut dao, &mut model);
assert_eq!(d.verdict(), Verdict::Rejected(why), "{name}");
assert_eq!(
(d.candidate_rev(), d.candidate_round(), d.panel()[0].candidate),
(None, None, None),
"{name}"
);
assert_eq!(d.candidate_inventory_digest(), None, "{name}");
let want = match &edit {
Ok(a) => Some(format!("{:016x}", fnv1a64(a.as_bytes()))),
Err(_) => None,
};
assert_eq!(d.edit_answer_digest(), want.as_deref(), "{name}");
assert_eq!(
dao.run.rounds().unwrap().len(),
2,
"{name}: no candidate round"
);
assert_eq!(model.seen.len(), 1, "{name}: no encode prompt was asked");
let dir = dao.root.join("cycle-1");
assert!(!dir.join("inventory.cc").exists(), "{name}");
assert!(!dir.join("inventory.cc.lock").exists(), "{name}");
assert_eq!(head_rev(&dao.root), 3, "{name}");
let tag = if why == Why::EditParse {
"edit-parse"
} else {
"edit-admit"
};
assert_eq!(
dao.run.cycle_line(&d),
format!(
"cycle 1 rev 3->none en-US base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag} lowering=tool-less-relay",
d.panel()[0].base.composite
),
"{name}"
);
assert_eq!(dao.run.decisions().unwrap(), [d], "{name}");
// The next cycle is told the last edit was rejected and asks afresh, in
// a directory of its own.
assert_eq!(dao.run.next_cycle_dir().unwrap(), dao.root.join("cycle-2"));
model.edit = Some(Ok(ONE_CONCEPT.into()));
let d2 = run_cycle(&mut dao, &mut model);
assert_eq!(d2.k(), 2, "{name}");
assert!(
mint_texts(&model)[1].contains("last edit: rejected;"),
"{name}"
);
}
}
#[test]
fn cycle_needs_a_floor() {
let none = |dao: &mut Dao, model: &mut FakeModel| {
let before = std::fs::read(dao.root.join("run.cc")).unwrap();
let err = cycle(&mut dao.run, model, &Scorer::new(Bag)).err();
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), before);
assert!(dao.run.decisions().unwrap().is_empty());
assert!(!dao.root.join("cycle-1").exists());
assert!(model.seen.is_empty(), "no prompt before the floor");
err
};
let mut dao = dao();
let mut model = cycle_fake(ONE_CONCEPT);
assert_eq!(none(&mut dao, &mut model), Some(RunError::NoFloor));
let mut one = fake(&degraded_answers());
recorded(round(&mut dao.run, &mut one, &Scorer::new(Bag)).unwrap());
assert_eq!(none(&mut dao, &mut model), Some(RunError::NoFloor));
assert_eq!(
RunError::NoFloor.to_string(),
"the run has no noise floor: record two rounds first"
);
// A panel language with no live Validation item never reaches a round:
// `create` refuses it (open_refuses_a_bad_panel).
assert_eq!(
RunError::NoValidation.to_string(),
"a reading has no validation item"
);
}
#[test]
fn a_failed_candidate_round_writes_nothing() {
let mut dao = dao();
two_rounds(&mut dao);
let run_cc = std::fs::read(dao.root.join("run.cc")).unwrap();
let mut model = cycle_fake(ONE_CONCEPT);
assert_eq!(
cycle(&mut dao.run, &mut model, &Scorer::new(Down)).err(),
Some(RunError::Embed)
);
assert_eq!(std::fs::read(dao.root.join("run.cc")).unwrap(), run_cc);
assert_eq!(dao.run.rounds().unwrap().len(), 2);
assert!(dao.run.decisions().unwrap().is_empty());
assert!(!dao.root.join("cycle-1/packets.cc").exists());
assert_eq!(head_rev(&dao.root), 3);
// The rerun starts clean from the head and decides on round 3.
let d = run_cycle(&mut dao, &mut model);
assert_eq!((d.k(), d.candidate_round()), (1, Some(3)));
assert_eq!(d.verdict(), Verdict::Accepted);
}
#[test]
fn a_changed_head_is_drift() {
let revision = |rev: u32| weksa::InventoryRevision {
n: rev + 1,
parent: Some(rev),
changes: vec![],
frozen: false,
};
let append = |file: &Path, rev: u32| {
weksa::InventoryStore::open(file)
.unwrap()
.admit_revision(revision(rev), vec![])
.unwrap();
};
// Before any accept the head is the run's copy.
let mut dao = dao();
append(&dao.root.join("inventory.cc"), 3);
assert_eq!(
RunStores::open(&dao.root).err(),
Some(RunError::Drift(DriftOf::Inventory))
);
// After an accept it is the cycle's file; the first copy is no longer read.
let mut dao2 = self::dao();
two_rounds(&mut dao2);
let mut model = cycle_fake(ONE_CONCEPT);
assert_eq!(
run_cycle(&mut dao2, &mut model).verdict(),
Verdict::Accepted
);
drop(dao2.run);
assert!(RunStores::open(&dao2.root).is_ok());
append(&dao2.root.join("inventory.cc"), 3);
assert!(
RunStores::open(&dao2.root).is_ok(),
"the superseded copy is not the head"
);
append(&dao2.root.join("cycle-1/inventory.cc"), 4);
assert_eq!(
RunStores::open(&dao2.root).err(),
Some(RunError::Drift(DriftOf::Inventory))
);
dao.run = RunStores::create(
&dao.root.join("again"),
"x",
RunPaths {
corpus: &dao.corpus,
inventory: &dao.inventory,
entities: &dao.entities,
},
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
assert_eq!(
dao.run.record().inventory,
std::fs::canonicalize(&dao.inventory)
.unwrap()
.to_str()
.unwrap()
);
}
/// Tripwires over the source, with the limits stated: they read text, so they
/// catch a spelling and not an alias.
#[test]
fn source_tripwires() {
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
let read = |rel: &str| std::fs::read_to_string(src.join(rel)).unwrap();
for file in ["run.rs", "model.rs", "encode.rs"] {
let text = read(file);
for banned in ["Split::Test", "Checkpoint"] {
assert!(!text.contains(banned), "{file} names {banned}");
}
}
let mut all = Vec::new();
let mut stack = vec![src.clone()];
while let Some(dir) = stack.pop() {
for entry in std::fs::read_dir(dir).unwrap() {
let path = entry.unwrap().path();
if path.is_dir() {
stack.push(path);
} else if path.extension().is_some_and(|e| e == "rs") {
all.push((
path.strip_prefix(&src)
.unwrap()
.to_str()
.unwrap()
.to_string(),
std::fs::read_to_string(&path).unwrap(),
));
}
}
}
for (file, text) in &all {
let code: String = text
.lines()
.filter(|l| !l.trim_start().starts_with("//"))
.collect::<Vec<_>>()
.join("\n");
if file != "main.rs" {
assert!(!code.contains("std::process"), "{file}");
}
// The Ollama reply is the other xenos boundary: JSON read from a server.
if file != "encode.rs" && file != "score/embed.rs" {
assert!(!code.contains("serde_json::from_str"), "{file}");
}
if file != "context.rs" {
assert!(!code.contains("Prompt {"), "{file}");
}
}
let context = read("context.rs");
assert!(!context.contains("pub fn from_corpus") && !context.contains("pub fn for_round"));
// The editor's prompt reads Train items, an inventory and an EditFeedback:
// no round record, decision, held-out mean or refusal count is named there.
for banned in [
"RoundRecord",
"Decision",
"SplitMean",
"Aggregate",
"validation_mean",
"validation_refused",
] {
assert!(!context.contains(banned), "context.rs names {banned}");
}
// `EditFeedback::none` is the one public constructor; Decision has none.
assert!(!context.contains("pub fn new"));
let run = read("run.rs");
let code: Vec<&str> = run
.lines()
.filter(|l| !l.trim_start().starts_with("//"))
.collect();
let count = |needle: &str| code.iter().filter(|l| l.contains(needle)).count();
assert!(!run.contains("pub fn new"));
// The round does not call the template table: the model lowers.
assert_eq!(
code.iter()
.filter(|l| l.contains("lower(&") && !l.contains("Prompt::lower("))
.count(),
0,
"run.rs calls the table"
);
assert_eq!(count("crate::lower::lower"), 0, "run.rs imports the table");
assert_eq!(
count("Prompt::lower("),
2,
"two lowering call sites: the round's and the stance ablation's replay of it"
);
// Prompt::lower takes the view, the item's key, the inventory and the entity
// port: no corpus item, train set, offer or store.
let start = context.find("pub fn lower(").unwrap();
let signature = &context[start..start + context[start..].find(") -> ").unwrap()];
for banned in ["CorpusItem,", "CorpusItem)", "TrainItems", "EncoderOffer", "Store"] {
assert!(!signature.contains(banned), "Prompt::lower takes {banned}");
}
assert!(signature.contains("&LoweringInput") && signature.contains("&CorpusItemId"));
assert_eq!(count("admit_revision("), 1, "one door call");
assert_eq!(
count("std::fs::copy"),
1,
"the cycle's copy; create's is copy_new"
);
assert_eq!(
count("Path::new(&record.inventory)"),
1,
"the source path is read once"
);
assert_eq!(
count("InventoryStore::open"),
2,
"the head and the cycle's file"
);
// An entry draft carries no id or revision field.
let encode = read("encode.rs");
let entry = encode
.split("pub struct EntryDraft {")
.nth(1)
.and_then(|t| t.split('}').next())
.unwrap();
for banned in ["id", "introduced_rev", "retired_rev"] {
assert!(
!entry
.split_whitespace()
.any(|w| w.trim_end_matches(':') == banned),
"EntryDraft has a {banned} field"
);
}
}
// ---- the lowerer ----
/// Lowercase letters and digits only: what survives any re-spacing or re-casing.
fn squashed(text: &str) -> String {
text.chars()
.filter(|c| c.is_alphanumeric())
.flat_map(char::to_lowercase)
.collect()
}
#[test]
fn the_lowerer_sees_no_item_text() {
let canary_key = "item:canary:1";
let mut dao = dao_over(|corpus, entities| {
let entities = weksa::EntityStore::open(entities).unwrap();
let mut store = weksa::CorpusStore::open(corpus).unwrap();
let mut item = store.get("item:daodejing:2").unwrap().unwrap();
item.id = canary_key.into();
item.text = "CANARYVALIDATIONTEXT of the zebra".into();
item.names = vec![];
item.prior_turns = vec![];
store.add(item.clone(), &entities).unwrap();
// A held-out ja-JP line too: the panel is en-US, so it is never offered.
item.id = "item:canary:2".into();
item.text = "CANARYJAPANESE 検証テキスト".into();
item.language = "ja-JP".into();
store.add(item, &entities).unwrap();
});
let mut model = fake(&good_answers());
model.answers.insert(canary_key.into(), json(&draft_of(2)));
model.lowered.insert(canary_key.into(), "CANARYLOWERED".into());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let corpus = weksa::CorpusStore::open(&dao.corpus).unwrap();
let items = corpus.all().unwrap();
assert!(items.len() >= 7, "the canaries and the five Legge items");
let packet_keys: Vec<String> = PacketStore::open(&dao.root.join("round-1/packets.cc"))
.unwrap()
.all()
.unwrap()
.iter()
.map(|p| p.id.to_string())
.collect();
assert_eq!(packet_keys.len(), 4);
let lowering: Vec<&(String, String)> = model
.seen
.iter()
.filter(|(_, t)| t.starts_with(LOWER_INSTRUCTIONS))
.collect();
assert_eq!(lowering.len(), 4, "one lowering per admitted packet");
for (digest, text) in &lowering {
let flat = squashed(text);
for item in &items {
// Any split: Train, Validation and Test text alike.
assert!(!text.contains(&item.text), "{}", item.id);
assert!(!flat.contains(&squashed(&item.text)), "{}", item.id);
for needle in [item.id.as_str(), item.group.as_str(), item.speaker.as_str()] {
assert!(!text.contains(needle), "{needle}");
}
}
for key in &packet_keys {
assert!(!text.contains(key.as_str()), "{key}");
}
assert!(!text.to_lowercase().contains("canary"));
assert!(!text.contains("confidence"));
// Its one call is its item.
let call = r.calls().iter().find(|c| &c.prompt == digest).unwrap();
assert_eq!(call.items.len(), 1);
assert!(call.items[0].starts_with("item:"));
}
// The encode and lowering calls name the same four items.
let keys = |lowering: bool| {
let mut keys: Vec<&str> = r
.calls()
.iter()
.filter(|c| {
lowering
== model.seen.iter().any(|(d, t)| {
d == &c.prompt && t.starts_with(LOWER_INSTRUCTIONS)
})
})
.map(|c| c.items[0].as_str())
.collect();
keys.sort_unstable();
keys
};
assert_eq!(keys(true), keys(false));
}
#[test]
fn lowering_refusals_count_at_lower() {
let run = |raw: Result<String, PortError>| {
let mut dao = dao();
let mut model = fake(&good_answers());
model.raw_lowering = Some(raw);
recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap())
};
for (what, raw) in [
("non-JSON", Ok("The sage empties the minds.".to_string())),
("unknown key", Ok(r#"{"text":"a","trace":"b"}"#.to_string())),
("missing text", Ok("{}".to_string())),
("empty text", Ok(r#"{"text":""}"#.to_string())),
("blank text", Ok("{\"text\":\" \\n \"}".to_string())),
("non-UTF-8", Err(PortError::Io(std::io::ErrorKind::InvalidData))),
("NUL", Ok(serde_json::json!({ "text": "The sage\u{0}acts." }).to_string())),
("bell", Ok(serde_json::json!({ "text": "The sage \u{7}acts." }).to_string())),
("carriage return", Ok(serde_json::json!({ "text": "The sage\racts." }).to_string())),
(
"over length in bytes, within it in chars",
Ok(serde_json::json!({ "text": "\u{e9}".repeat(weksa::encode::MAX_LOWERING_BYTES / 2 + 1) })
.to_string()),
),
(
"over length",
Ok(serde_json::json!({ "text": "a".repeat(weksa::encode::MAX_LOWERING_BYTES + 1) })
.to_string()),
),
] {
let r = run(raw);
assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Lower), "{what}");
assert!(
r.train()[0].packet.is_some(),
"{what}: the admitted packet is kept on the Train row"
);
let v = r.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (0, 0, 2), "{what}");
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (0, 0), "{what}");
let m = &r.language("en-US").unwrap().validation_mean;
assert_eq!((m.count, m.chrf), (2, Some(0.0)), "{what}");
}
}
#[test]
fn an_added_entry_can_be_lowered() {
// The edit adds a concept; the candidate encode answer cites it. The
// candidate's lowering prompt lists it and the item scores: the gain path a
// table with no row for it blocked.
let mut dao = dao();
two_rounds(&mut dao);
let mut cited = draft_of(1);
cited.referents[0].concept = "concept:28".parse().unwrap();
let mut model = cycle_fake(ONE_CONCEPT);
model
.candidate
.as_mut()
.unwrap()
.1
.insert("item:daodejing:1".into(), json(&cited));
run_cycle(&mut dao, &mut model);
let candidate_lowerings: Vec<&str> = model
.seen
.iter()
.map(|(_, t)| t.as_str())
.filter(|t| t.starts_with(LOWER_INSTRUCTIONS) && t.contains("\n\ninventory rev:4\n"))
.collect();
assert_eq!(candidate_lowerings.len(), 3);
for text in &candidate_lowerings {
assert!(text.contains("\nconcept:28: a general meaning the Dao names"));
}
assert_eq!(
candidate_lowerings
.iter()
.filter(|t| t.contains(r#""concept":"concept:28""#))
.count(),
1,
"exactly the item that cites the new concept"
);
let rounds = dao.run.rounds().unwrap();
let candidate = rounds.last().unwrap();
assert_eq!(candidate.inventory_rev(), 4);
assert!(matches!(
candidate.train()[0].outcome,
Outcome::Scored { .. }
));
}
#[test]
fn a_summary_line_says_how_lowerings_were_answered() {
use weksa::run::LowererReach;
assert_eq!("briefed-subagent".parse(), Ok(LowererReach::BriefedSubagent));
assert_eq!("tool-less-relay".parse(), Ok(LowererReach::ToolLessRelay));
assert!("Tool-Less-Relay".parse::<LowererReach>().is_err());
let dao = dao();
assert_eq!(
dao.run.labelled("round 1"),
"round 1 lowering=tool-less-relay"
);
// A briefed-subagent run is pinned as one, in the record that reopens, and
// its lines say its numbers are smoke-round only.
let root = dao.root.join("briefed");
let paths = RunPaths {
corpus: &dao.corpus,
inventory: &dao.inventory,
entities: &dao.entities,
};
RunStores::create(
&root,
"b",
paths,
"fake-model",
"bag",
LowererReach::BriefedSubagent,
ModelFamily::Unrecorded,
&["en-US".to_string()],
)
.unwrap();
let run = RunStores::open(&root).unwrap();
assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent);
assert_eq!(
run.labelled("cycle 1"),
"cycle 1 lowering=briefed-subagent (smoke-round only)"
);
}
#[test]
fn lowering_text_at_the_bound_is_scored_and_line_breaks_are_text() {
for text in [
"a".repeat(weksa::encode::MAX_LOWERING_BYTES),
"\u{e9}".repeat(weksa::encode::MAX_LOWERING_BYTES / 2),
"The sage acts.\n\tThe sage rests.".to_string(),
] {
let mut dao = dao();
let mut model = fake(&good_answers());
model.raw_lowering = Some(Ok(serde_json::json!({ "text": text }).to_string()));
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(
(r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored),
(1, 2),
"{}",
text.len()
);
assert_eq!(r.language("en-US").unwrap().validation_refused, Default::default());
}
}
/// Two canary items, one Train and one Validation, whose speaker is an entity
/// with forms only in another language; each encodes to a packet citing it.
#[test]
fn a_lowering_prompt_that_cannot_be_built_refuses_at_lower_and_keeps_the_packet() {
use weksa::entity::{Entity, EntityOrigin, LanguageForms};
use weksa::packet::EntityId;
let id = EntityId::parse("entity:sagexx").unwrap();
let mut dao = dao_over(|corpus, entities| {
let mut entity_store = weksa::EntityStore::open(entities).unwrap();
entity_store
.put(Entity {
id: EntityId::parse("entity:sagexx").unwrap(),
forms: vec![LanguageForms {
language: "sa-Latn".into(),
forms: vec!["Sagexa".into()],
pronunciation: None,
}],
origin: EntityOrigin::Cast {
persona: "sage".into(),
fixture: "f.yaml".into(),
},
})
.unwrap();
let mut store = weksa::CorpusStore::open(corpus).unwrap();
for (from, key) in [
("item:daodejing:1", "item:cite:1"),
("item:daodejing:2", "item:cite:2"),
] {
let mut item = store.get(from).unwrap().unwrap();
item.id = key.into();
item.text = format!("A held-apart sentence for {key}.");
item.speaker = EntityId::parse("entity:sagexx").unwrap();
item.names = vec![];
item.prior_turns = vec![];
store.add(item, &entity_store).unwrap();
}
});
let citing = |n: usize| {
let mut d = draft_of(n);
d.referents[0].entity = Some(id.clone());
json(&d)
};
let mut model = fake(&good_answers());
model.answers.insert("item:cite:1".into(), citing(1));
model.answers.insert("item:cite:2".into(), citing(2));
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
// Admitted, so the packet is on the Train row; refused at Lower, so no score.
let row = r.train().iter().find(|row| row.item == "item:cite:1").unwrap();
assert_eq!(row.outcome, Outcome::Refused(Stage::Lower));
let kept = row.packet.as_deref().expect("the admitted packet's key is kept");
let packets = PacketStore::open(&dao.root.join("round-1/packets.cc")).unwrap();
assert!(packets.get(kept).unwrap().is_some());
let v = r.language("en-US").unwrap().validation_refused;
assert_eq!((v.parse, v.admit, v.lower), (0, 0, 1));
assert_eq!((r.language("en-US").unwrap().train_scored, r.language("en-US").unwrap().validation_scored), (1, 2));
// Five items encoded, three lowering prompts built and asked.
assert_eq!(r.calls().len(), 5 + 3);
}
#[test]
fn a_port_failure_in_the_lowering_phase_discards_the_packets() {
let mut dao = dao();
let mut model = fake(&good_answers());
model.raw_lowering = Some(Err(PortError::Io(std::io::ErrorKind::PermissionDenied)));
assert_eq!(
round(&mut dao.run, &mut model, &Scorer::new(Bag)).err(),
Some(RunError::Io(std::io::ErrorKind::PermissionDenied))
);
assert!(!dao.root.join("round-1/packets.cc").exists());
assert!(dao.run.rounds().unwrap().is_empty());
}
#[test]
fn views_with_one_meaning_share_one_unanswered_lowering_prompt() {
let mut dao = dao();
// Items 1 and 3 encode to the same meaning; item 2 differs.
let same = json(&draft_of(1));
let expected = prompts(&dao.run);
let round1 = dao.root.join("round-1");
std::fs::create_dir_all(&round1).unwrap();
for (key, p) in &expected {
let answer = match key.as_str() {
"item:daodejing:2" => json(&draft_of(2)),
_ => same.clone(),
};
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1.clone(), "fake-model".into());
let RoundOutcome::Unanswered { prompts: asked } =
round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()
else {
panic!("answered lowering from nothing")
};
assert_eq!(asked.len(), 2, "{asked:?}");
let mut unique = asked.clone();
unique.sort();
unique.dedup();
assert_eq!(unique.len(), 2);
}
/// Several refusals under one rule in one packet: the item still counts once.
fn every_concept_unknown(d: &mut PacketDraft) {
for r in &mut d.referents {
r.concept = "concept:999".parse().unwrap();
}
for p in &mut d.predications {
p.concept = "concept:999".parse().unwrap();
}
}
#[test]
fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() {
use weksa::packet::EntityId;
let mut dao = dao();
let empty = |d: &mut PacketDraft| {
d.referents.clear();
d.predications.clear();
d.relations.clear();
};
let nobody = |d: &mut PacketDraft| {
d.referents[0].entity = Some("entity:nobody".parse::<EntityId>().unwrap());
};
let mut model = fake(&[
(1, edited(1, nobody)),
(2, edited(2, every_concept_unknown)),
(3, edited(3, empty)),
]);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let rules: Vec<(&str, u32)> = r
.admit_rules()
.iter()
.map(|c| (c.rule.as_str(), c.count))
.collect();
assert_eq!(rules, [("UnknownEntity", 1), ("V2", 1), ("V5", 1)]);
let line = dao.run.round_line(&r);
assert!(
line.ends_with("admit-rules=UnknownEntityx1,V2x1,V5x1 lowering=tool-less-relay"),
"{line}"
);
// A round nothing refused names no rule and prints none.
let mut dao = self::dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert!(r.admit_rules().is_empty());
assert!(!dao.run.round_line(&r).contains("admit-rules"));
}
/// Item 1's good draft as JSON, changed by `change`, as the model's answer.
fn mangled(change: impl FnOnce(&mut serde_json::Value)) -> String {
let mut v: serde_json::Value = serde_json::from_str(&json(&draft_of(1))).unwrap();
change(&mut v);
v.to_string()
}
/// The round record of a Dao run whose item 1 is answered with `answer` and
/// the rest well.
fn round_with_item_1(answer: String) -> (Dao, RoundRecord) {
let mut dao = dao();
let mut answers = good_answers();
answers[0].1 = answer;
let mut model = fake(&answers);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
(dao, r)
}
fn reasons(r: &RoundRecord) -> Vec<(&str, u32)> {
r.parse_reasons()
.iter()
.map(|c| (c.rule.as_str(), c.count))
.collect()
}
#[test]
fn parse_reasons_by_class() {
let good = json(&draft_of(1));
let truncated = good[..good.len() / 2].to_string();
let cases: Vec<(&str, String)> = vec![
("syntax", "not json at all".to_string()),
("eof", truncated),
(
"invalid-type",
mangled(|v| v["referents"][0]["concept"] = serde_json::json!(7)),
),
(
"bad-form",
mangled(|v| v["referents"][0]["concept"] = serde_json::json!("entity:x")),
),
(
"unknown-field",
mangled(|v| v["surprise"] = serde_json::json!(1)),
),
(
"missing-field",
mangled(|v| {
v.as_object_mut().unwrap().remove("referents");
}),
),
];
for (code, answer) in cases {
let (dao, r) = round_with_item_1(answer);
assert_eq!(reasons(&r), [(code, 1)], "{code}");
assert_eq!(
r.train()[0].outcome,
Outcome::Refused(Stage::Parse),
"{code}"
);
let line = dao.run.round_line(&r);
assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}");
}
// A round nothing refused at Parse carries and prints none.
let (dao, r) = round_with_item_1(good);
assert!(r.parse_reasons().is_empty());
assert!(!dao.run.round_line(&r).contains("parse-reasons"));
}
#[test]
fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() {
let mut dao = dao();
let expected = prompts(&dao.run);
let answers = keyed(&good_answers());
let round1 = dao.root.join("round-1");
std::fs::create_dir_all(&round1).unwrap();
for (key, p) in &expected {
let bytes = match key.as_str() {
// Train: not UTF-8, so no answer parsed from the file.
"item:daodejing:1" => vec![0xff, 0xfe],
// Validation: not JSON.
"item:daodejing:2" => b"nope".to_vec(),
_ => answers[key].clone().into_bytes(),
};
std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap();
}
let table = table_english();
for (key, p) in lowering_prompts(&dao, dao.run.inventory())
.iter()
.filter(|(key, _)| key == "item:daodejing:3")
{
let answer = serde_json::json!({ "text": table[key] }).to_string();
std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap();
}
let mut files = AnswerFiles::new(round1, "fake-model".into());
let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap());
assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]);
}
#[test]
fn parse_reasons_never_echo_answer() {
let canary = "CANARY-7f3a-held-out";
// The canary sits where serde would quote it: an unknown field name, an
// invalid value, and the raw text of a non-JSON answer.
let answers = [
mangled(|v| v[canary] = serde_json::json!(1)),
mangled(|v| v["referents"][0]["concept"] = serde_json::json!(canary)),
format!("{canary} is not json"),
];
for answer in answers {
// Item 2 is Validation: the held-out item carries the canary.
let mut dao = dao();
let mut a = good_answers();
a[1].1 = answer;
let mut model = fake(&a);
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
assert_eq!(r.language("en-US").unwrap().validation_refused.parse, 1);
let line = dao.run.round_line(&r);
assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}");
let bytes = std::fs::read(dao.root.join("run.cc")).unwrap();
assert!(!bytes.windows(6).any(|w| w == b"CANARY"));
assert!(!format!("{r:?}").contains("CANARY"));
}
}
/// A run record as written before `answerer_family` (key 13) and `panel` (key 14) existed.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_run")]
struct PreFamilyRun {
#[cultcache(key = 0)]
label: String,
#[cultcache(key = 1)]
corpus: String,
#[cultcache(key = 2)]
inventory: String,
#[cultcache(key = 3)]
entities: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
prompt_digest: String,
#[cultcache(key = 8)]
model: String,
#[cultcache(key = 9)]
embedder: String,
#[cultcache(key = 10)]
chrf_version: String,
#[cultcache(key = 11)]
inventory_digest: String,
#[cultcache(key = 12)]
lowerer_reach: weksa::run::LowererReach,
}
/// A round record as written with the pooled means (keys 11 to 15) and before
/// `parse_reasons`, `own_family` and `languages`.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_round")]
struct LegacyRound {
#[cultcache(key = 0)]
n: u32,
#[cultcache(key = 1)]
inventory_rev: u32,
#[cultcache(key = 2)]
prompt_digest: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
model: String,
#[cultcache(key = 7)]
embedder: String,
#[cultcache(key = 8)]
chrf_version: String,
#[cultcache(key = 9)]
calls: Vec<weksa::run::Call>,
#[cultcache(key = 10)]
train: Vec<weksa::run::TrainRow>,
#[cultcache(key = 11)]
train_mean: weksa::score::SplitMean,
#[cultcache(key = 12)]
validation_mean: weksa::score::SplitMean,
#[cultcache(key = 13)]
validation_refused: weksa::run::RefusalCounts,
#[cultcache(key = 14)]
train_scored: u32,
#[cultcache(key = 15)]
validation_scored: u32,
#[cultcache(key = 16)]
inventory_digest: String,
#[cultcache(key = 17)]
cycle: Option<u32>,
}
#[test]
fn a_run_from_before_does_not_open() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let rec = dao.run.record().clone();
let root = dao.root.clone();
let legacy = LegacyRound {
n: r.n(),
inventory_rev: r.inventory_rev(),
prompt_digest: r.prompt_digest().into(),
corpus_digest: r.corpus_digest().into(),
entity_digest: r.entity_digest().into(),
model: r.model().into(),
embedder: r.embedder().into(),
chrf_version: r.chrf_version().into(),
calls: r.calls().to_vec(),
train: r.train().to_vec(),
train_mean: r.language("en-US").unwrap().train_mean.clone(),
validation_mean: r.language("en-US").unwrap().validation_mean.clone(),
validation_refused: r.language("en-US").unwrap().validation_refused,
train_scored: r.language("en-US").unwrap().train_scored,
validation_scored: r.language("en-US").unwrap().validation_scored,
inventory_digest: r.inventory_digest().into(),
cycle: r.cycle(),
};
let run = PreFamilyRun {
label: rec.label.clone(),
corpus: rec.corpus.clone(),
inventory: rec.inventory.clone(),
entities: rec.entities.clone(),
corpus_digest: rec.corpus_digest.clone(),
entity_digest: rec.entity_digest.clone(),
prompt_digest: rec.prompt_digest.clone(),
model: rec.model.clone(),
embedder: rec.embedder.clone(),
chrf_version: rec.chrf_version.clone(),
inventory_digest: rec.inventory_digest.clone(),
lowerer_reach: rec.lowerer_reach,
};
drop(dao.run);
let store = || SingleFileMessagePackBackingStore::new(root.join("run.cc"));
// A run record without a panel (no key 14) is a scratch run off git.
{
let mut cache = CultCache::new();
cache.register_entry_type::<PreFamilyRun>().unwrap();
cache.register_entry_type::<LegacyRound>().unwrap();
cache.add_generic_backing_store(store()).unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &run).unwrap();
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
}
assert_eq!(RunStores::open(&root).err(), Some(RunError::Store));
// A round with the pooled means (keys 11 to 15) and no languages is
// unreadable, under a record that has its panel.
let _ = std::fs::remove_file(root.join("run.cc"));
{
let mut cache = CultCache::new();
cache.register_entry_type::<RunRecord>().unwrap();
cache.register_entry_type::<LegacyRound>().unwrap();
cache.add_generic_backing_store(store()).unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &rec).unwrap();
cache.put("run:dao:round:1".to_string(), &legacy).unwrap();
}
// No round of that run can be read: it stops at its first round or cycle.
let reopened = RunStores::open(&root).unwrap();
assert!(reopened.rounds().is_err());
}
#[test]
fn the_published_run_and_round_schemas_require_none_of_the_new_fields() {
let published = weksa::schema::published();
let schema = |needle: &str| -> serde_json::Value {
let (_, json) = published
.iter()
.find(|(name, _)| name.contains(needle))
.unwrap();
serde_json::from_str(json).unwrap()
};
for (needle, fields) in [
("loop_round", &["parse_reasons", "own_family"][..]),
("loop_run", &["answerer_family"][..]),
] {
let s = schema(needle);
let required = s["required"].as_array().unwrap();
for f in fields {
assert!(s["properties"][f].is_object(), "{needle} {f}");
assert!(!required.iter().any(|r| r == f), "{needle} {f} is required");
}
}
}
/// What a recall prompt shows of its item: the text after `opening: `.
fn shown_opening(prompt: &str) -> &str {
prompt.rsplit_once("opening: ").expect("a recall prompt has an opening").1
}
/// The item's text after the opening the prompt showed.
fn held_back<'a>(text: &'a str, prompt: &str) -> &'a str {
let opening = shown_opening(prompt);
text.strip_prefix(opening)
.unwrap_or_else(|| panic!("the opening is a prefix of the item"))
.trim()
}
/// Answers a recall probe from the corpus: the item's own held-back half (its
/// text after the opening the prompt showed), a fixed wrong text, or junk.
struct Reciter {
corpus: weksa::store::CorpusStore,
/// Given the item's held-back half.
answer: fn(&str) -> String,
/// An item whose text holds this gets an answer file that is not UTF-8.
unreadable: Option<&'static str>,
seen: Vec<String>,
}
impl ModelPort for Reciter {
fn model(&self) -> &str {
"reciter"
}
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
self.seen.push(prompt.text().to_string());
let key = &prompt.item_keys()[0];
let item = self.corpus.get(key).unwrap().unwrap();
if self.unreadable.is_some_and(|t| item.text.contains(t)) {
return Err(PortError::Io(std::io::ErrorKind::InvalidData));
}
Ok((self.answer)(held_back(&item.text, prompt.text())))
}
}
fn second_half(held_back: &str) -> String {
serde_json::json!({ "text": held_back }).to_string()
}
fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter {
Reciter {
corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(),
answer,
unreadable: None,
seen: vec![],
}
}
#[test]
fn a_recall_probe_asks_for_the_second_half_of_each_live_item_and_never_shows_it() {
use weksa::run::RecallOutcome;
let dao = dao();
let mut perfect = reciter(&dao, second_half);
let RecallOutcome::Read(reading) = dao.run.recall(&mut perfect).unwrap() else {
panic!("every prompt was answered");
};
let r = reading.languages[0].clone();
// Items 1 (Train) and 2, 3 (Validation) are live; 4 and 5 are not offered.
assert_eq!((r.n, r.answered), (3, 3));
assert!((r.chrf - 100.0).abs() < 1e-9, "{}", r.chrf);
assert_eq!(perfect.seen.len(), 3);
for (text, key) in perfect.seen.iter().zip(["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"]) {
let item = perfect.corpus.get(key).unwrap().unwrap();
// The opening is a proper prefix of the item, cut at a word boundary
// near its middle; the rest is held back.
let head = shown_opening(text);
let tail = held_back(&item.text, text);
assert!(!head.is_empty() && !tail.is_empty(), "{key}");
let squeeze = |t: &str| t.split_whitespace().collect::<String>();
assert_eq!(squeeze(&item.text), squeeze(&format!("{head}{tail}")), "{key}: head and tail are the item");
let n = item.text.len();
assert!((n / 4..=3 * n / 4).contains(&head.len()), "{key}: near the middle");
assert!(!text.contains(tail), "{key}: the held half is not shown");
}
}
#[test]
fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits() {
use weksa::run::RecallOutcome;
let dao = dao();
let mut wrong = reciter(&dao, |_| r#"{"text":"zzz qqq xxx"}"#.to_string());
let RecallOutcome::Read(reading) = dao.run.recall(&mut wrong).unwrap() else {
panic!("answered");
};
let r = reading.languages[0].clone();
assert_eq!((r.n, r.answered), (3, 3));
assert!(r.chrf < 10.0, "{}", r.chrf);
let mut junk = reciter(&dao, |_| "not json".to_string());
let RecallOutcome::Read(reading) = dao.run.recall(&mut junk).unwrap() else {
panic!("answered");
};
let r = reading.languages[0].clone();
assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0));
// With no answers on disk the files port waits on all three prompts.
let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into());
let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else {
panic!("nobody answered");
};
assert_eq!(prompts.len(), 3);
assert!(dao.run.recall_dir().join(format!("{}.prompt", prompts[0])).exists());
}
#[test]
fn the_published_round_schema_does_not_require_a_field_older_records_lack() {
let (_, json) = weksa::schema::published()
.into_iter()
.find(|(name, _)| name.contains("loop_round"))
.expect("the round schema is published");
let schema: serde_json::Value = serde_json::from_str(&json).unwrap();
let required = schema["required"].as_array().unwrap();
assert!(required.iter().any(|r| r == "inventory_digest"));
assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}");
assert!(schema["properties"]["admit_rules"].is_object());
}
#[test]
fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdrawn_item_is_not_probed() {
use weksa::corpus::Source;
use weksa::run::RecallOutcome;
let dao = dao_over(|corpus, entities| {
let entities = weksa::EntityStore::open(entities).unwrap();
let mut store = weksa::CorpusStore::open(corpus).unwrap();
// Another source: probed too, but it names no work.
let mut written = store.get("item:daodejing:2").unwrap().unwrap();
written.id = "item:written:1".into();
written.text = "A line of some other source with words enough to split.".into();
written.names = vec![];
written.prior_turns = vec![];
written.source = Source::OperatorWritten;
store.add(written, &entities).unwrap();
store.withdraw("item:daodejing:3").unwrap();
});
let both = |dao: &Dao, unreadable: Option<&'static str>| {
let mut m = reciter(dao, second_half);
m.unreadable = unreadable;
let RecallOutcome::Read(reading) = dao.run.recall(&mut m).unwrap() else {
panic!("answered");
};
reading.languages[0].clone()
};
let full = both(&dao, None);
assert_eq!(
(full.n, full.answered),
(3, 3),
"the written item is probed, the withdrawn one is not"
);
// One of three unreadable: the mean falls to two thirds.
let part = both(&dao, Some("Heaven is long-enduring"));
assert_eq!((part.n, part.answered), (3, 2));
assert!((part.chrf - full.chrf * 2.0 / 3.0).abs() < 1e-9, "{}", part.chrf);
// A public-domain prompt names its work; any other says it is not published.
let mut m = reciter(&dao, second_half);
dao.run.recall(&mut m).unwrap();
let named = m.seen.iter().filter(|t| t.contains("translator: James Legge")).count();
let unnamed = m.seen.iter().filter(|t| t.contains("work: not published")).count();
assert_eq!((named, unnamed), (2, 1));
}
/// Answers every prompt with text that is not a draft, so a round measures only
/// what it counts before admission.
struct NotADraft;
impl weksa::model::ModelPort for NotADraft {
fn model(&self) -> &str {
"not-a-draft"
}
fn answer(&mut self, _: &weksa::context::Prompt) -> Result<String, weksa::model::PortError> {
Ok("not a draft".to_string())
}
}
struct OneBag;
impl weksa::score::Embedder for OneBag {
fn model(&self) -> &str {
"bag"
}
fn embed(&self, texts: &[&str]) -> anyhow::Result<Vec<Vec<f32>>> {
Ok(texts.iter().map(|_| vec![1.0]).collect())
}
}
/// A corpus of every shape a family count must tell apart, and the own-family
/// count of a round over it by an answerer of `family`.
fn mixed_family_round(family: ModelFamily) -> u32 {
use common::{cast, group_in_bucket, item_in};
use weksa::corpus::Source;
use weksa::run::{LowererReach, RoundOutcome, round};
let mut w = world();
let written = |family| Source::ModelWritten {
family,
generator: None,
prompt_digest: None,
};
let public_domain = Source::PublicDomain {
work: "w".into(),
author: "a".into(),
edition_url: "https://example.org/e".into(),
licence: "PD".into(),
jurisdiction: "US".into(),
translator: "t".into(),
edition_year: 1900,
};
let (train, validation, test) = (
group_in_bucket(10),
group_in_bucket(50),
group_in_bucket(90),
);
let items = [
// Offered, Anthropic: two of them count for Anthropic.
("item:mix:1", &train, written(ModelFamily::Anthropic)),
("item:mix:2", &validation, written(ModelFamily::Anthropic)),
// Offered, other families or no model at all: never Anthropic's.
("item:mix:3", &train, written(ModelFamily::OpenAi)),
("item:mix:4", &train, written(ModelFamily::Unrecorded)),
("item:mix:5", &train, public_domain),
("item:mix:6", &train, cast()),
// Model-written in a Test group: in no split, never offered.
("item:mix:7", &test, written(ModelFamily::Anthropic)),
// Withdrawn after admission: never offered.
("item:mix:8", &train, written(ModelFamily::Anthropic)),
];
let entities = common::entities_in(w.dir.path());
for (key, group, source) in items {
let text = format!("Line {key}, distinct.");
w.corpus
.add(item_in(key, &text, group, source), &entities)
.unwrap();
}
w.corpus.withdraw("item:mix:8").unwrap();
let dir = w.dir.path().to_path_buf();
for name in ["c.cc", "i.cc", "entities.cc"] {
assert!(dir.join(name).exists(), "{name}");
}
let mut run = RunStores::create(
&dir.join("run"),
"mix",
RunPaths {
corpus: &dir.join("c.cc"),
inventory: &dir.join("i.cc"),
entities: &dir.join("entities.cc"),
},
"not-a-draft",
"bag",
LowererReach::ToolLessRelay,
family,
&["en-US".to_string()],
)
.unwrap();
let scorer = weksa::score::Scorer::new(OneBag);
match round(&mut run, &mut NotADraft, &scorer).unwrap() {
RoundOutcome::Recorded(r) => {
// Eight items were offered: the world's two cast items and mix 1 to 6;
// the Test-group and withdrawn items were not.
assert_eq!(
r.parse_reasons().iter().map(|c| c.count).sum::<u32>(),
8,
"offered items"
);
r.own_family()
}
RoundOutcome::Unanswered { .. } => panic!("answered"),
}
}
#[test]
fn own_family_counts_offered_items_of_the_answerer_family() {
// Two live Train/Validation Anthropic items; the Test-group and withdrawn
// Anthropic items are not offered, so they are not counted.
assert_eq!(mixed_family_round(ModelFamily::Anthropic), 2);
assert_eq!(mixed_family_round(ModelFamily::OpenAi), 1);
// Google wrote nothing here; an unrecorded answerer counts no item, though
// an offered item is itself unrecorded.
assert_eq!(mixed_family_round(ModelFamily::Google), 0);
assert_eq!(mixed_family_round(ModelFamily::Unrecorded), 0);
}
// ---- stance ablation (stance-to-lowerer) ----
/// A lowerer by function: the text it answers for an item key and the prompt
/// text, recording every prompt it was shown.
struct Lowerer {
f: fn(&str, &str) -> String,
seen: Vec<(String, String, String)>,
}
impl Lowerer {
fn new(f: fn(&str, &str) -> String) -> Self {
Self { f, seen: vec![] }
}
/// The digests asked, once each, in the order first asked.
fn digests(&self) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
for (d, ..) in &self.seen {
if !out.contains(d) {
out.push(d.clone());
}
}
out
}
}
impl ModelPort for Lowerer {
fn model(&self) -> &str {
"fake-model"
}
fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
let key = prompt.item_keys()[0].clone();
self.seen.push((prompt.digest(), key.clone(), prompt.text().to_string()));
Ok(serde_json::json!({ "text": (self.f)(&key, prompt.text()) }).to_string())
}
}
const WRITTEN: &str = "item:written:1";
fn english(key: &str) -> String {
let mut table = table_english();
table.insert(
WRITTEN.into(),
"Heaven endures through the long years without end.".into(),
);
table[key].clone()
}
/// The packet line's stance, as the prompt prints it: the packet line is one
/// JSON object ending in the stance.
fn stance_of(text: &str) -> String {
let line = text.lines().find(|l| l.starts_with("packet: ")).unwrap();
line[line.find("\"stance\":").unwrap()..].to_string()
}
/// Says the same thing whatever the stance.
fn blind(key: &str, _text: &str) -> String {
english(key)
}
/// Appends the stance it was shown.
fn reader(key: &str, text: &str) -> String {
format!("{} {}", english(key), stance_of(text))
}
/// Does better with the item's own stance: the real sentence for it, nonsense
/// for any other, empty included.
fn helped(key: &str, text: &str) -> String {
let r800 = r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}}]}}"#;
let own = match key {
"item:daodejing:3" => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":800}},{"feature":"feature:16","value":{"Grade":300}}]}}"#,
WRITTEN => r#""stance":{"features":[{"feature":"feature:18","value":{"Grade":500}}]}}"#,
_ => r800,
};
if stance_of(text) == own { english(key) } else { "zzz qqq xxx".into() }
}
fn grade_stance(features: &[(&str, u16)]) -> weksa::packet::Stance {
weksa::packet::Stance {
features: features
.iter()
.map(|(f, v)| weksa::packet::FeatureValue {
feature: f.parse().unwrap(),
value: weksa::packet::FeatValue::Grade(common::grade(*v)),
})
.collect(),
}
}
/// A Dao run with a third Validation item (`item:written:1`, key order after
/// items 2 and 3) and round 1 recorded over it, every item encoded to its Dao
/// packet. Stances: item 1 restraint 800; item 2 restraint 800; item 3
/// restraint 800 and affection 300; the written item restraint 500.
fn recorded_dao(edit: impl FnOnce(&mut HashMap<String, String>)) -> (Dao, RoundRecord) {
let mut dao = dao_over(|corpus, entities| {
let entities = weksa::EntityStore::open(entities).unwrap();
let mut store = CorpusStore::open(corpus).unwrap();
let mut written = store.get("item:daodejing:2").unwrap().unwrap();
written.id = WRITTEN.into();
written.text = "Heaven endures through the long years without end.".into();
written.source = weksa::corpus::Source::OperatorWritten;
store.add(written, &entities).unwrap();
});
let mut model = fake(&good_answers());
let mut extra = draft_of(2);
extra.referents.truncate(1);
extra.predications.truncate(1);
extra.stance = grade_stance(&[("feature:18", 500)]);
model.answers.insert(WRITTEN.into(), json(&extra));
edit(&mut model.answers);
model.lowered.insert(WRITTEN.into(), english(WRITTEN));
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
(dao, r)
}
fn ablate(
dao: &Dao,
recorded: &mut dyn ModelPort,
model: &mut dyn ModelPort,
) -> Result<weksa::run::AblationOutcome, RunError> {
dao.run.stance_ablation(1, recorded, model, &Scorer::new(Bag))
}
fn read(
outcome: weksa::run::AblationOutcome,
) -> Vec<(weksa::Split, weksa::run::AblationReading)> {
match outcome {
weksa::run::AblationOutcome::Read(r) => r.into_iter().map(|(_, split, x)| (split, x)).collect(),
weksa::run::AblationOutcome::Unanswered { prompts } => panic!("unanswered {prompts:?}"),
}
}
#[test]
fn recorded_answers_read_and_never_write() {
use std::io::ErrorKind;
use weksa::model::RecordedAnswers;
let p = prompt();
let dir = tempfile::tempdir().unwrap();
let mut port = RecordedAnswers::new(dir.path().to_path_buf(), "answerer".into());
assert_eq!(port.model(), "answerer");
// A miss is Unanswered and the directory is as it was.
assert_eq!(port.answer(&p), Err(PortError::Unanswered));
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
// A directory that does not exist is a miss too, and is not made.
let mut absent = RecordedAnswers::new(dir.path().join("nope"), "m".into());
assert_eq!(absent.answer(&p), Err(PortError::Unanswered));
assert!(!dir.path().join("nope").exists());
// A hit is the file's text as written.
let file = dir.path().join(format!("{}.answer", p.digest()));
std::fs::write(&file, " {\"x\": 1}\n").unwrap();
assert_eq!(port.answer(&p).unwrap(), " {\"x\": 1}\n");
std::fs::write(&file, [0xff, 0xfe]).unwrap();
assert_eq!(port.answer(&p), Err(PortError::Io(ErrorKind::InvalidData)));
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1);
}
#[test]
fn a_stance_ablation_asks_replay_drop_and_permute_for_train_and_validation_only() {
let (dao, r) = recorded_dao(|_| {});
let round_digests: Vec<String> = r.calls()[4..].iter().map(|c| c.prompt.clone()).collect();
assert_eq!(round_digests.len(), 4, "the round lowered four items");
let mut recorded_port = Lowerer::new(reader);
let mut model = Lowerer::new(reader);
read(ablate(&dao, &mut recorded_port, &mut model).unwrap());
// The base prompts are the round's own, byte for byte.
let recorded_digests = recorded_port.digests();
assert_eq!(recorded_digests, round_digests);
let asked = model.digests();
for d in &round_digests {
assert!(asked.contains(d), "the replay asks the round's prompt {d}");
}
// Per item: replay and drop for each; permute for each Validation item, and
// none for the lone Train item.
let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count();
assert_eq!(per_key("item:daodejing:1"), 2);
for key in ["item:daodejing:2", "item:daodejing:3", WRITTEN] {
assert_eq!(per_key(key), 3, "{key}");
}
// Test items are never asked.
for (_, key, text) in &model.seen {
assert!(key != "item:daodejing:4" && key != "item:daodejing:5", "{key}");
assert!(!text.contains("item:daodejing:4"));
}
// A drop prompt prints the empty stance, a replay the item's own.
let empty = model
.seen
.iter()
.filter(|s| stance_of(&s.2) == "\"stance\":{\"features\":[]}}")
.count();
assert_eq!(empty, 4, "one drop per item");
}
#[test]
fn a_lowerer_blind_to_stance_reads_no_movement_and_one_that_reads_it_does() {
let (dao, _) = recorded_dao(|_| {});
let one = |f: fn(&str, &str) -> String| {
let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f));
read(ablate(&dao, &mut rec, &mut model).unwrap())
};
for (split, r) in one(blind) {
assert_eq!(r.unusable, 0);
assert_eq!((r.drop.moved, r.permute.moved), (0, 0), "{split:?}: {}", r.line());
assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9);
assert!((r.drop.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9);
if r.permute.n > 0 {
assert!((r.permute.chrf.unwrap() - r.replay.chrf.unwrap()).abs() < 1e-9);
}
assert_eq!(r.drop.ref_gain_chrf, Some(0.0));
}
let reading = one(reader);
let (train, validation) = (&reading[0].1, &reading[1].1);
assert_eq!((reading[0].0, reading[1].0), (weksa::Split::Train, weksa::Split::Validation));
assert_eq!((train.n, validation.n), (1, 3));
assert_eq!((train.drop.n, train.drop.moved), (1, 1));
assert_eq!((train.permute.n, train.permute.moved), (0, 0));
assert_eq!((validation.drop.n, validation.drop.moved), (3, 3));
assert_eq!((validation.permute.n, validation.permute.moved), (3, 3));
// A lone Train item has no permute: no measurement, not total disagreement.
assert_eq!(
train.permute,
weksa::run::VariantReading::default(),
"{}",
train.line()
);
assert!(
train.line().contains(
"| permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none"
),
"{}",
train.line()
);
assert!((validation.replay.chrf.unwrap() - 100.0).abs() < 1e-9);
assert!(validation.drop.chrf.unwrap() < validation.replay.chrf.unwrap());
assert!(validation.permute.chrf.unwrap() < validation.replay.chrf.unwrap());
}
#[test]
fn the_item_s_own_stance_helps_fidelity_only_for_a_lowerer_that_uses_it() {
let (dao, _) = recorded_dao(|_| {});
let one = |f: fn(&str, &str) -> String| {
let (mut rec, mut model) = (Lowerer::new(f), Lowerer::new(f));
read(ablate(&dao, &mut rec, &mut model).unwrap())
};
for (split, r) in one(helped) {
assert!(r.drop.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line());
assert!(r.drop.ref_gain_meaning.unwrap() > 0.0, "{split:?}: {}", r.line());
if r.permute.n > 0 {
assert!(r.permute.ref_gain_chrf.unwrap() > 10.0, "{split:?}: {}", r.line());
}
assert_eq!(r.drop.moved, r.drop.n);
}
for (split, r) in one(blind) {
assert_eq!((r.drop.ref_gain_chrf, r.drop.ref_gain_meaning), (Some(0.0), Some(0.0)), "{split:?}");
}
}
#[test]
fn a_packet_without_stance_asks_no_drop_and_a_test_packet_in_the_round_is_never_asked() {
let (dao, _) = recorded_dao(|answers| {
let mut d = draft_of(3);
d.stance = Default::default();
answers.insert("item:daodejing:3".into(), json(&d));
});
// A packet for a Test item, put into the round's own store the way no round
// would: the ablation reads splits from the corpus, not from the store.
{
let corpus = CorpusStore::open(&dao.corpus).unwrap();
let inv = dao.run.inventory();
let mut packet = packets().remove(3);
packet.inventory_rev = 3;
PacketStore::open(&dao.root.join("round-1/packets.cc"))
.unwrap()
.admit(packet, inv, &corpus, dao.run.entities())
.unwrap();
}
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
read(ablate(&dao, &mut rec, &mut model).unwrap());
let bare = "\"stance\":{\"features\":[]}}";
let drops_of_3 = model
.seen
.iter()
.filter(|s| s.1 == "item:daodejing:3" && stance_of(&s.2) == bare)
.count();
// Its replay already shows the empty stance; there is no drop beside it.
assert_eq!(drops_of_3, 1);
assert!(model.seen.iter().all(|s| s.1 != "item:daodejing:4"), "a Test item is never asked");
assert!(rec.seen.iter().all(|s| s.1 != "item:daodejing:4"));
}
#[test]
fn permute_takes_the_next_items_stance_in_its_split_and_skips_equal_ones() {
let (dao, _) = recorded_dao(|_| {});
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
read(ablate(&dao, &mut rec, &mut model).unwrap());
// The stance each Validation item's permute prompt shows: the next item's
// in key order (2, 3, written:1), cyclic; item 1 is alone in Train.
let stance = |text: &str| stance_of(text);
let of = |key: &str| -> Vec<String> {
model
.seen
.iter()
.filter(|s| s.1 == key)
.map(|s| stance(&s.2))
.collect()
};
let (s2, s3, sw) = (
stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:2").unwrap().2),
stance(&rec.seen.iter().find(|s| s.1 == "item:daodejing:3").unwrap().2),
stance(&rec.seen.iter().find(|s| s.1 == WRITTEN).unwrap().2),
);
assert!(s2 != s3 && s3 != sw && sw != s2);
assert!(of("item:daodejing:2").contains(&s3));
assert!(of("item:daodejing:3").contains(&sw));
assert!(of(WRITTEN).contains(&s2), "the last wraps to the first");
assert!(!of("item:daodejing:2").contains(&sw), "not the previous or the other");
// Item 1 is alone in Train: it never takes a stance across splits.
assert_eq!(of("item:daodejing:1").len(), 2, "replay and drop only");
// Items 2 and 3 with equal stances: item 2's permute is its own prompt, so
// it asks none; the others still do.
let (dao, _) = recorded_dao(|answers| {
let mut d = draft_of(3);
d.stance = draft_of(2).stance;
answers.insert("item:daodejing:3".into(), json(&d));
});
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
let r = read(ablate(&dao, &mut rec, &mut model).unwrap());
let per_key = |key: &str| model.seen.iter().filter(|s| s.1 == key).count();
assert_eq!(per_key("item:daodejing:2"), 2, "replay and drop: its next holds the stance it has");
assert_eq!(per_key("item:daodejing:3"), 3);
assert_eq!(per_key(WRITTEN), 3);
assert_eq!(r[1].1.permute.n, 2, "{}", r[1].1.line());
}
#[test]
fn a_stance_ablation_records_nothing_and_writes_nothing_in_its_round() {
use weksa::model::RecordedAnswers;
let (dao, r) = recorded_dao(|_| {});
let round1 = dao.root.join("round-1");
for c in &r.calls()[4..] {
std::fs::write(
round1.join(format!("{}.answer", c.prompt)),
serde_json::json!({ "text": english(&c.items[0]) }).to_string(),
)
.unwrap();
}
let snapshot = |dir: &Path| -> Vec<(String, Vec<u8>)> {
let mut files: Vec<_> = std::fs::read_dir(dir)
.unwrap()
.map(|e| {
let e = e.unwrap();
let bytes = if e.path().is_dir() { vec![] } else { std::fs::read(e.path()).unwrap() };
(e.file_name().to_string_lossy().into_owned(), bytes)
})
.collect();
files.sort();
files.retain(|f| f.0 != "ablation-1");
files
};
let before = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap());
let go = || {
let mut rec = RecordedAnswers::new(round1.clone(), "fake-model".into());
let mut files = weksa::model::AnswerFiles::new(dao.run.ablation_dir(1), "fake-model".into());
// Nothing answers yet: the replay and variant prompts are written.
dao.run.stance_ablation(1, &mut rec, &mut files, &Scorer::new(Bag))
};
let weksa::run::AblationOutcome::Unanswered { prompts } = go().unwrap() else {
panic!("nobody answered")
};
// Four replays (the round's own prompts), four drops, three permutes (item 1
// is alone in Train).
assert_eq!(prompts.len(), 4 + 4 + 3, "{prompts:?}");
for d in &prompts {
assert!(dao.run.ablation_dir(1).join(format!("{d}.prompt")).is_file());
}
// Answer every ask from a fake and read; then again over the same files.
for d in &prompts {
let text = std::fs::read_to_string(dao.run.ablation_dir(1).join(format!("{d}.prompt"))).unwrap();
std::fs::write(
dao.run.ablation_dir(1).join(format!("{d}.answer")),
serde_json::json!({ "text": format!("answer {}", &text[..8]) }).to_string(),
)
.unwrap();
}
let first = read(go().unwrap());
let again = read(go().unwrap());
assert_eq!(first, again);
let after = (snapshot(&round1), snapshot(&dao.root), dao.run.rounds().unwrap(), dao.run.decisions().unwrap());
assert!(before.0 == after.0, "round-1 is unchanged byte for byte");
assert!(before.1 == after.1, "the run root's files are unchanged");
assert_eq!(before.2, after.2);
assert_eq!(before.3, after.3);
}
#[test]
fn a_stance_ablation_refuses_a_cycle_round_and_missing_round_answers() {
use weksa::model::RecordedAnswers;
let (dao, _) = recorded_dao(|_| {});
// No round-2, and round 1 holds no answer files.
let mut model = Lowerer::new(reader);
let mut rec = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
assert!(matches!(
dao.run.stance_ablation(2, &mut rec, &mut model, &Scorer::new(Bag)),
Err(RunError::Malformed("round"))
));
let Err(missing) = ablate(&dao, &mut rec, &mut model) else {
panic!("a missing round answer is refused")
};
assert!(matches!(missing, RunError::Unrecorded));
// It says what happened, not that a store would not read.
let said = missing.to_string();
assert!(said.contains("holds no answer") && said.contains("digest") && said.contains("head inventory"), "{said}");
assert!(!said.contains("would not read"), "{said}");
assert!(
std::fs::read_dir(dao.root.join("round-1"))
.unwrap()
.all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".prompt")),
"no prompt is written into the round"
);
// Drift is refused as a round refuses it.
struct Named(Lowerer);
impl ModelPort for Named {
fn model(&self) -> &str {
"other-model"
}
fn answer(&mut self, p: &Prompt) -> Result<String, PortError> {
self.0.answer(p)
}
}
let mut named = Named(Lowerer::new(reader));
let mut rec2 = Lowerer::new(reader);
assert!(matches!(
ablate(&dao, &mut rec2, &mut named),
Err(RunError::Drift(DriftOf::Model))
));
assert!(matches!(
dao.run.stance_ablation(1, &mut rec2, &mut Lowerer::new(reader), &Scorer::new(OtherEmbedder)),
Err(RunError::Drift(DriftOf::Embedder))
));
}
#[test]
fn a_stance_ablation_refuses_a_cycle_round() {
let mut dao = dao();
two_rounds(&mut dao);
run_cycle(&mut dao, &mut cycle_fake(ONE_CONCEPT));
let rounds = dao.run.rounds().unwrap();
let candidate = rounds.iter().find(|r| r.cycle().is_some()).expect("the cycle's round");
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(reader));
assert!(matches!(
dao.run.stance_ablation(candidate.n(), &mut rec, &mut model, &Scorer::new(Bag)),
Err(RunError::Malformed("round"))
));
assert!(model.seen.is_empty() && rec.seen.is_empty());
}
/// The real sentence for the item's own stance, an empty text (not a usable
/// lowering) for any other.
fn mute_unless_own(key: &str, text: &str) -> String {
let said = helped(key, text);
if said == "zzz qqq xxx" { String::new() } else { said }
}
#[test]
fn unusable_variant_answers_are_counted_and_read_as_none_not_zero() {
let (dao, _) = recorded_dao(|_| {});
let (mut rec, mut model) = (Lowerer::new(mute_unless_own), Lowerer::new(mute_unless_own));
let reading = read(ablate(&dao, &mut rec, &mut model).unwrap());
let (train, validation) = (&reading[0].1, &reading[1].1);
// Every variant answer is unusable: Train has one drop, Validation three
// drops and three permutes. The items themselves (replay) still read.
assert_eq!((train.n, train.unusable), (1, 1));
assert_eq!((validation.n, validation.unusable), (3, 6));
for r in [train, validation] {
assert!((r.replay.chrf.unwrap() - 100.0).abs() < 1e-9);
assert_eq!(r.drop.n, 0);
assert_eq!(r.permute.n, 0);
assert_eq!(r.drop, weksa::run::VariantReading::default());
assert_eq!(r.permute, weksa::run::VariantReading::default());
}
assert_eq!(
train.line(),
"ablation n=1 unusable=1 replay chrf=100.00 meaning=1.000 | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none"
);
}
#[test]
fn the_line_names_each_variant_with_its_own_n_and_the_splits_n() {
use weksa::run::{AblationReading, Agreement, VariantReading};
let reading = AblationReading {
n: 7,
unusable: 2,
replay: Agreement { chrf: Some(91.5), meaning: Some(0.9) },
drop: VariantReading {
n: 5,
chrf: Some(80.25),
meaning: Some(0.8),
moved: 4,
ref_gain_chrf: Some(3.5),
ref_gain_meaning: Some(-0.125),
},
permute: VariantReading {
n: 3,
chrf: Some(70.0),
meaning: None,
moved: 1,
ref_gain_chrf: None,
ref_gain_meaning: Some(0.5),
},
};
assert_eq!(
reading.line(),
"ablation n=7 unusable=2 replay chrf=91.50 meaning=0.900 | drop n=5 chrf=80.25 meaning=0.800 moved=4 ref_gain_chrf=3.50 ref_gain_meaning=-0.125 | permute n=3 chrf=70.00 meaning=none moved=1 ref_gain_chrf=none ref_gain_meaning=0.500"
);
}
fn nothing_usable(_: &str, _: &str) -> String {
String::new()
}
#[test]
fn a_split_where_no_replay_is_usable_reads_none_throughout() {
let (dao, _) = recorded_dao(|_| {});
let (mut rec, mut model) = (Lowerer::new(reader), Lowerer::new(nothing_usable));
let reading = read(ablate(&dao, &mut rec, &mut model).unwrap());
// Train: the item and its drop; Validation: three items, three drops, three permutes.
assert_eq!((reading[0].1.n, reading[0].1.unusable), (0, 2));
assert_eq!((reading[1].1.n, reading[1].1.unusable), (0, 9));
for (_, r) in &reading {
assert_eq!(r.replay, weksa::run::Agreement::default());
assert_eq!(r.drop, weksa::run::VariantReading::default());
assert_eq!(r.permute, weksa::run::VariantReading::default());
}
assert_eq!(
reading[0].1.line(),
"ablation n=0 unusable=2 replay chrf=none meaning=none | drop n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none"
);
}
// ---- the panel: each item is lowered in its own language ----
const JA_TEXTS: [(&str, &str, usize); 3] = [
("item:daodejing:11", "賢者は民の心を空にし、腹を満たし、志を弱め、骨を強くする。", 1),
("item:daodejing:12", "天は長く、地は久しい。", 2),
("item:daodejing:13", "上善は水のようである。", 3),
];
/// The Dao corpus with a ja-JP twin of each live item (same group, so the same
/// split) and a pt-BR twin of each; Laozi, the speaker the packets cite, gains
/// `ja` as a ja-JP form when given.
fn add_languages(corpus: &Path, entities: &Path, ja: Option<&str>) {
let mut entities = weksa::EntityStore::open(entities).unwrap();
if let Some(form) = ja {
let mut laozi = entities.get(&common::eid("laozi")).unwrap().unwrap();
laozi.forms.push(weksa::LanguageForms {
language: "ja-JP".into(),
forms: vec![form.into()],
pronunciation: None,
});
entities.put(laozi).unwrap();
}
let mut store = CorpusStore::open(corpus).unwrap();
for (key, text, twin) in JA_TEXTS {
let mut item = store.get(&format!("item:daodejing:{twin}")).unwrap().unwrap();
item.id = key.into();
item.text = text.into();
item.language = "ja-JP".into();
item.names = vec![];
item.prior_turns = vec![];
store.add(item, &entities).unwrap();
let mut pt = store.get(&format!("item:daodejing:{twin}")).unwrap().unwrap();
pt.id = format!("item:daodejing:2{twin}");
pt.text = format!("PORTUGUESEMARK {twin} o sabio esvazia os coracoes");
pt.language = "pt-BR".into();
pt.names = vec![];
pt.prior_turns = vec![];
store.add(pt, &entities).unwrap();
}
}
/// The Dao draft of `item`, its first referent citing the speaker, Laozi.
fn citing_laozi(item: usize) -> String {
let mut v = serde_json::to_value(draft_of(item)).unwrap();
v["referents"][0]["entity"] = serde_json::json!("entity:laozi");
v.to_string()
}
/// Encode answers for the en-US and ja-JP items, all citing the sage.
fn bilingual_answers() -> HashMap<String, String> {
let mut answers = HashMap::new();
for n in 1..=3 {
answers.insert(format!("item:daodejing:{n}"), citing_laozi(n));
answers.insert(format!("item:daodejing:1{n}"), citing_laozi(n));
}
answers
}
fn bilingual_fake() -> FakeModel {
let mut model = fake(&[]);
model.answers = bilingual_answers();
for (key, text, _) in JA_TEXTS {
model.lowered.insert(key.into(), text.into());
}
model
}
fn bilingual(ja: Option<&'static str>) -> Dao {
dao_panel(
|corpus, entities| add_languages(corpus, entities, ja),
&["ja-JP", "en-US"],
)
}
const LAOZI_JA: Option<&str> = Some("老子");
fn lowering_texts(model: &FakeModel) -> Vec<&str> {
model
.seen
.iter()
.map(|(_, t)| t.as_str())
.filter(|t| t.starts_with(LOWER_INSTRUCTIONS))
.collect()
}
#[test]
fn each_item_lowers_in_its_own_language() {
let mut dao = bilingual(LAOZI_JA);
assert_eq!(dao.run.record().panel, ["en-US", "ja-JP"], "sorted at open");
let mut model = bilingual_fake();
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let prompts = lowering_texts(&model);
assert_eq!(prompts.len(), 6, "three items in each language");
let (mut ja, mut en) = (0, 0);
for text in prompts {
if text.contains("\n\nlanguage: ja-JP\n\n") {
ja += 1;
assert!(text.contains(r#"{"entity:laozi":["老子"]}"#), "{text}");
assert!(!text.contains("Laozi"), "{text}");
} else {
en += 1;
assert!(text.contains("\n\nlanguage: en-US\n\n"), "{text}");
assert!(text.contains(r#"{"entity:laozi":["Laozi"]}"#), "{text}");
assert!(!text.contains("老子"), "{text}");
}
}
assert_eq!((ja, en), (3, 3));
// No lowering prompt, in either language, holds any item's text, key or group.
let corpus = CorpusStore::open(&dao.corpus).unwrap();
for text in lowering_texts(&model) {
for item in corpus.all().unwrap() {
assert!(!text.contains(&item.text), "{}", item.id);
assert!(!text.contains(item.id.as_str()), "{}", item.id);
assert!(!text.contains(item.group.as_str()), "{}", item.id);
}
}
// A ja-JP item is scored against its own text: the lowering that is the
// text reads as a perfect one.
let ja = r.language("ja-JP").unwrap();
assert_eq!((ja.train_scored, ja.validation_scored), (1, 2));
assert_eq!(ja.validation_mean.chrf, Some(100.0));
assert_eq!(ja.train_mean.meaning, Some(1.0));
}
#[test]
fn a_cited_entity_without_a_form_in_the_items_language_refuses_at_lower() {
let mut dao = bilingual(None);
let mut model = bilingual_fake();
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let ja = r.language("ja-JP").unwrap();
assert_eq!(ja.validation_refused.lower, 2);
assert_eq!((ja.train_scored, ja.validation_scored), (0, 0));
let refused: Vec<_> = r
.train()
.iter()
.filter(|row| matches!(row.outcome, Outcome::Refused(Stage::Lower)))
.collect();
assert_eq!(refused.len(), 1, "the ja-JP Train item");
assert_eq!(refused[0].item, "item:daodejing:11");
let en = r.language("en-US").unwrap();
assert_eq!((en.train_scored, en.validation_scored), (1, 2));
assert_eq!(en.validation_refused, weksa::run::RefusalCounts::default());
assert_eq!(lowering_texts(&model).len(), 3, "no prompt for a form that is missing");
}
#[test]
fn the_panel_filters_every_prompt() {
let mut dao = bilingual(LAOZI_JA);
let mut model = bilingual_fake();
// Two rounds, a cycle (the editor's prompt), a recall: every kind of prompt.
let mut last = None;
for _ in 0..2 {
last = Some(recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()));
}
let tags = |r: &RoundRecord| -> Vec<String> {
r.languages().iter().map(|l| l.language.clone()).collect()
};
assert_eq!(tags(&last.unwrap()), ["en-US", "ja-JP"]);
model.candidate = Some((4, bilingual_answers()));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
run_cycle(&mut dao, &mut model);
let mut recite = reciter(&dao, second_half);
let reading = match dao.run.recall(&mut recite).unwrap() {
weksa::run::RecallOutcome::Read(r) => r,
_ => panic!("answered"),
};
let langs: Vec<&str> = reading.languages.iter().map(|l| l.language.as_str()).collect();
assert_eq!(langs, ["en-US", "ja-JP"]);
let shown = model.seen.iter().map(|(_, t)| t).chain(&recite.seen);
let mut count = 0;
for text in shown {
count += 1;
assert!(!text.contains("PORTUGUESE"), "a pt-BR item reached a prompt");
for n in 1..=3 {
assert!(!text.contains(&format!("item:daodejing:2{n}")));
}
}
assert!(count > 20, "{count}");
assert!(!mint_texts(&model).is_empty());
assert_eq!(dao.run.train().unwrap().len(), 2, "one Train item per panel language");
}
#[test]
fn open_refuses_a_bad_panel() {
let train_only = |corpus: &Path, entities: &Path| {
let entities = weksa::EntityStore::open(entities).unwrap();
let mut store = CorpusStore::open(corpus).unwrap();
let mut item = store.get("item:daodejing:1").unwrap().unwrap();
item.id = "item:daodejing:31".into();
item.text = "Un texte francais.".into();
item.language = "fr-FR".into();
item.names = vec![];
item.prior_turns = vec![];
store.add(item, &entities).unwrap();
};
let dao = dao_over(train_only);
let (corpus, inventory, entities) = (&dao.corpus, &dao.inventory, &dao.entities);
drop(dao.run);
let tag = |s: &str| s.to_string();
for panel in [
vec![],
vec![tag("en-US"), tag("en-US")],
vec![tag("en-US"), tag("not a tag!")],
// No live item at all, and a Train item without a Validation one.
vec![tag("en-US"), tag("de-DE")],
vec![tag("en-US"), tag("fr-FR")],
] {
let root = dao._dir.path().join(format!("panel-{}", panel.len()));
let created = RunStores::create(
&root,
"p",
RunPaths { corpus, inventory, entities },
"fake-model",
"bag",
weksa::run::LowererReach::ToolLessRelay,
ModelFamily::Unrecorded,
&panel,
);
let err = created.err();
assert_eq!(err, Some(RunError::Panel), "{panel:?}");
let shown = err.unwrap().to_string();
for t in &panel {
assert!(!shown.contains(t.as_str()), "{shown}");
}
assert!(!root.join("run.cc").exists(), "{panel:?}");
assert!(!root.join("inventory.cc").exists(), "{panel:?}");
}
}
#[test]
fn round_means_are_per_language() {
let mut dao = bilingual(LAOZI_JA);
let mut model = bilingual_fake();
// One ja-JP Validation item is not an encodable draft: it counts 0.
model.answers.insert("item:daodejing:13".into(), "not a draft".into());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let ja = r.language("ja-JP").unwrap();
// Validation: item 12 reads perfectly (chrF 100, meaning 1), item 13 is 0.
assert_eq!(ja.validation_mean.count, 2);
assert_eq!(ja.validation_mean.chrf, Some(50.0));
assert_eq!(ja.validation_mean.meaning, Some(0.5));
assert_eq!((ja.validation_scored, ja.validation_refused.parse), (1, 1));
assert_eq!(ja.train_mean.chrf, Some(100.0));
// en-US reads exactly as it does when it is the whole panel: the ja-JP
// items touch none of its numbers.
let mut alone = dao_over(|_, _| {});
let mut plain = fake(&good_answers());
let r2 = recorded(round(&mut alone.run, &mut plain, &Scorer::new(Bag)).unwrap());
assert_eq!(r.language("en-US"), r2.language("en-US"));
assert_eq!(r.languages().len(), 2);
assert_eq!(r2.languages().len(), 1);
}
#[test]
fn floor_and_feedback_are_per_language() {
let mut dao = bilingual(LAOZI_JA);
// Round 1: everything lowers perfectly in ja-JP. Round 2: one ja-JP
// Validation item is refused. en-US reads the same twice.
let mut first = bilingual_fake();
recorded(round(&mut dao.run, &mut first, &Scorer::new(Bag)).unwrap());
let mut second = bilingual_fake();
second.answers.insert("item:daodejing:13".into(), "not a draft".into());
recorded(round(&mut dao.run, &mut second, &Scorer::new(Bag)).unwrap());
let mut model = bilingual_fake();
model.candidate = Some((4, bilingual_answers()));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
let d = run_cycle(&mut dao, &mut model);
let rows = d.panel();
assert_eq!(
rows.iter().map(|r| r.language.as_str()).collect::<Vec<_>>(),
["en-US", "ja-JP"]
);
let (en, ja) = (&rows[0], &rows[1]);
assert_eq!(en.floor.composite, 0.0, "en-US read the same twice");
assert_eq!(en.threshold, LADDER_STEP);
assert_eq!(ja.floor.chrf, 50.0);
assert_eq!(ja.floor.meaning, 0.5);
assert_eq!(ja.floor.composite, 0.5);
assert_eq!(ja.threshold, 0.5);
assert_eq!(ja.base.composite, 1.0);
// The editor's one held-out number is each language's head composite on
// the ladder, in panel order.
let rounded = |x: f64| (x / LADDER_STEP).round() * LADDER_STEP;
let line = format!(
"last edit: none; validation composite at head: en-US {:.3}, ja-JP {:.3}",
rounded(en.base.composite),
rounded(ja.base.composite)
);
let mint = mint_texts(&model);
assert!(mint[0].contains(&format!("\n\n{line}\n\n")), "{}", mint[0]);
}
#[test]
fn recall_probes_unspaced_lines() {
let dao = bilingual(LAOZI_JA);
let mut recite = reciter(&dao, second_half);
let reading = match dao.run.recall(&mut recite).unwrap() {
weksa::run::RecallOutcome::Read(r) => r,
_ => panic!("answered"),
};
let ja = reading.languages.iter().find(|l| l.language == "ja-JP").unwrap();
// Three lines written without spaces were skipped before.
assert_eq!((ja.n, ja.answered), (3, 3));
assert!((ja.chrf - 100.0).abs() < 1e-9);
let corpus = CorpusStore::open(&dao.corpus).unwrap();
for (key, ja_text, _) in JA_TEXTS {
let prompt = recite
.seen
.iter()
.find(|t| shown_opening(t).chars().next() == ja_text.chars().next())
.unwrap();
let item = corpus.get(key).unwrap().unwrap();
let (head, tail) = (shown_opening(prompt), held_back(&item.text, prompt));
assert!(!head.is_empty() && !tail.is_empty());
assert_eq!(format!("{head}{tail}"), item.text, "no spaces to lose");
let n = item.text.len();
assert!((n / 4..=3 * n / 4).contains(&head.len()), "{key}: near the byte middle");
assert!(!prompt.contains(tail));
}
// The unrelated-text floor was measured for English only: its clause
// prints once, on the en-US segment.
let line = reading.line();
assert_eq!(line.matches("unrelated English").count(), 1, "{line}");
let (first, second) = line.split_once(" | ").unwrap();
assert!(first.starts_with("recall en-US") && first.contains("unrelated English"), "{line}");
assert!(second.starts_with("recall ja-JP") && !second.contains("unrelated"), "{line}");
// en-US still splits between words.
let en = reading.languages.iter().find(|l| l.language == "en-US").unwrap();
assert_eq!((en.n, en.answered), (3, 3));
for key in ["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"] {
let item = corpus.get(key).unwrap().unwrap();
let prompt = recite
.seen
.iter()
.find(|t| t.contains("opening: ") && item.text.starts_with(shown_opening(t)))
.unwrap();
let head = shown_opening(prompt);
let next = item.text[head.len()..].chars().next().unwrap();
assert!(!next.is_alphanumeric(), "{key}: cut inside a word");
}
}
#[test]
fn offer_by_key_is_the_panels_too() {
let dao = bilingual(LAOZI_JA);
assert!(dao.run.offer("item:daodejing:12").unwrap().is_ok(), "ja-JP is in the panel");
assert!(dao.run.offer("item:daodejing:2").unwrap().is_ok());
for pt in ["item:daodejing:21", "item:daodejing:22", "item:daodejing:23"] {
assert!(dao.run.offer(pt).is_err(), "{pt}: pt-BR is outside the panel");
}
}
#[test]
fn a_cycle_decides_each_language_on_its_own_floor() {
let mut dao = bilingual(LAOZI_JA);
// en-US reads the same twice (floor 0); ja-JP loses a Validation item in
// round 2 (a wide floor).
let mut first = bilingual_fake();
recorded(round(&mut dao.run, &mut first, &Scorer::new(Bag)).unwrap());
let mut second = bilingual_fake();
second.answers.insert("item:daodejing:13".into(), "not a draft".into());
recorded(round(&mut dao.run, &mut second, &Scorer::new(Bag)).unwrap());
// The candidate degrades en-US a little and leaves ja-JP as it was.
let mut model = bilingual_fake();
model.candidate = Some((4, {
let mut candidate = bilingual_answers();
candidate.extend(keyed(&degraded_answers()));
candidate
}));
model.edit = Some(Ok(ONE_CONCEPT.to_string()));
let d = run_cycle(&mut dao, &mut model);
let (en, ja) = (&d.panel()[0], &d.panel()[1]);
assert_eq!((en.language.as_str(), ja.language.as_str()), ("en-US", "ja-JP"));
assert_eq!(en.floor.chrf, 0.0);
assert_eq!(ja.floor.chrf, 50.0);
// en-US fell by more than its own floor but by less than ja-JP's: the
// verdict is the rejection that en-US's floor makes, not NoGain.
let fall = en.base.chrf - en.candidate.unwrap().chrf;
assert!(fall > 1.0 && fall < ja.floor.chrf, "{fall}");
assert!(matches!(d.verdict(), Verdict::Rejected(Why::Regressed(_))), "{:?}", d.verdict());
}
#[test]
fn ablation_reads_each_language_apart() {
let mut dao = bilingual(LAOZI_JA);
let mut model = bilingual_fake();
recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let (mut rec, mut replay) = (bilingual_fake(), bilingual_fake());
let readings = match ablate(&dao, &mut rec, &mut replay).unwrap() {
weksa::run::AblationOutcome::Read(r) => r,
_ => panic!("answered"),
};
let shape: Vec<(&str, weksa::Split, u32)> = readings
.iter()
.map(|(l, split, r)| (l.as_str(), *split, r.n))
.collect();
use weksa::Split::{Train, Validation};
assert_eq!(
shape,
[("en-US", Train, 1), ("en-US", Validation, 2), ("ja-JP", Train, 1), ("ja-JP", Validation, 2)]
);
}