541 lines
18 KiB
Rust
541 lines
18 KiB
Rust
mod common;
|
|
|
|
use common::{cast, ent, entities_in, group_in_bucket, item_in, model, run_over, world};
|
|
use weksa::context::{
|
|
ENCODE_INSTRUCTIONS as ENCODE, EditFeedback, MINT_INSTRUCTIONS as MINT, OfferRefusal, Prompt,
|
|
};
|
|
use weksa::run::RunStores;
|
|
use weksa::{
|
|
CorpusItem, CorpusItemId, CorpusStore, NameSpan, PacketId, PacketStore, StoreError,
|
|
StoreRefusal, Turn,
|
|
};
|
|
|
|
struct Corpus {
|
|
_dir: tempfile::TempDir,
|
|
store: CorpusStore,
|
|
run: RunStores,
|
|
}
|
|
|
|
/// One live item per split, a withdrawn Train item and a model-written
|
|
/// Test-group item, each with a marker word unique to it.
|
|
fn corpus() -> Corpus {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let entities = entities_in(dir.path());
|
|
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
|
let mut put = |item: CorpusItem| {
|
|
let withdrawn = item.withdrawn;
|
|
let id = item.id.clone();
|
|
store
|
|
.add(
|
|
CorpusItem {
|
|
withdrawn: false,
|
|
..item
|
|
},
|
|
&entities,
|
|
)
|
|
.unwrap();
|
|
if withdrawn {
|
|
store.withdraw(&id).unwrap();
|
|
}
|
|
};
|
|
put(item_in(
|
|
"item:a:1",
|
|
"alphamarker speaks",
|
|
&group_in_bucket(10),
|
|
cast(),
|
|
));
|
|
put(item_in(
|
|
"item:b:1",
|
|
"betamarker speaks",
|
|
&group_in_bucket(50),
|
|
cast(),
|
|
));
|
|
put(item_in(
|
|
"item:c:1",
|
|
"gammamarker speaks",
|
|
&group_in_bucket(90),
|
|
cast(),
|
|
));
|
|
let mut gone = item_in(
|
|
"item:a:2",
|
|
"deltamarker speaks",
|
|
&group_in_bucket(10),
|
|
cast(),
|
|
);
|
|
gone.withdrawn = true;
|
|
put(gone);
|
|
put(item_in(
|
|
"item:c:2",
|
|
"epsilonmarker speaks",
|
|
&group_in_bucket(90),
|
|
model(),
|
|
));
|
|
let run = run_over(dir.path());
|
|
Corpus {
|
|
_dir: dir,
|
|
store,
|
|
run,
|
|
}
|
|
}
|
|
|
|
fn get(c: &Corpus, key: &str) -> CorpusItem {
|
|
c.store.get(key).unwrap().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn train_items_hold_only_live_train() {
|
|
let c = corpus();
|
|
let t = c.run.train().unwrap();
|
|
assert_eq!(t.keys().collect::<Vec<_>>(), ["item:a:1"]);
|
|
assert_eq!((t.len(), t.is_empty()), (1, false));
|
|
}
|
|
|
|
#[test]
|
|
fn train_items_are_in_key_order() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let entities = entities_in(dir.path());
|
|
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
|
for (key, text) in [
|
|
("item:z:1", "zed line"),
|
|
("item:a:1", "ay line"),
|
|
("item:m:1", "em line"),
|
|
] {
|
|
store
|
|
.add(item_in(key, text, &group_in_bucket(10), cast()), &entities)
|
|
.unwrap();
|
|
}
|
|
let t = run_over(dir.path()).train().unwrap();
|
|
assert_eq!(
|
|
t.keys().collect::<Vec<_>>(),
|
|
["item:a:1", "item:m:1", "item:z:1"]
|
|
);
|
|
assert_eq!((t.len(), t.is_empty()), (3, false));
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let empty = run_over(dir.path());
|
|
let none = empty.train().unwrap();
|
|
assert_eq!((none.len(), none.is_empty()), (0, true));
|
|
let (inv, fb) = (empty.inventory().inventory().unwrap(), EditFeedback::none());
|
|
assert_eq!(
|
|
Prompt::mint(&none, &inv, 0, &fb).text(),
|
|
format!("{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0")
|
|
);
|
|
assert!(Prompt::mint(&none, &inv, 0, &fb).item_keys().is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn mint_prompt_holds_train_text_only() {
|
|
let c = corpus();
|
|
let t = c.run.train().unwrap();
|
|
let inv = c.run.inventory().inventory().unwrap();
|
|
let p = Prompt::mint(&t, &inv, 4, &EditFeedback::none());
|
|
assert!(p.text().starts_with(MINT));
|
|
assert!(p.text().contains("alphamarker"));
|
|
for other in ["betamarker", "gammamarker", "deltamarker", "epsilonmarker"] {
|
|
assert!(!p.text().contains(other), "{other}");
|
|
}
|
|
assert_eq!(p.item_keys(), ["item:a:1"]);
|
|
}
|
|
|
|
#[test]
|
|
fn prompts_render_deterministically() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let entities = entities_in(dir.path());
|
|
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
|
let mut one = item_in("item:a:1", "Hello there.", &group_in_bucket(10), cast());
|
|
one.prior_turns = vec![Turn {
|
|
speaker: common::eid("brokkr"),
|
|
text: "Well?".into(),
|
|
}];
|
|
store.add(one, &entities).unwrap();
|
|
store
|
|
.add(
|
|
item_in("item:a:2", "Second line.", &group_in_bucket(10), cast()),
|
|
&entities,
|
|
)
|
|
.unwrap();
|
|
let run = run_over(dir.path());
|
|
let t = run.train().unwrap();
|
|
let inv = run.inventory().inventory().unwrap();
|
|
let fb = EditFeedback::none();
|
|
assert_eq!(
|
|
Prompt::mint(&t, &inv, 0, &fb).text(),
|
|
format!(
|
|
"{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0\n\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line."
|
|
)
|
|
);
|
|
assert_eq!(
|
|
Prompt::mint(&t, &inv, 0, &fb).item_keys(),
|
|
["item:a:1", "item:a:2"]
|
|
);
|
|
|
|
let offer = run.offer("item:a:2").unwrap().unwrap();
|
|
assert_eq!(
|
|
Prompt::encode(&offer, &t, &inv, 0).text(),
|
|
format!(
|
|
"{ENCODE}\n\n[item:a:2] entity:nibu: Second line.\n\nentities: entity:nibu\n\ninventory rev:0"
|
|
)
|
|
);
|
|
assert_eq!(
|
|
Prompt::encode(&offer, &t, &inv, 0).item_keys(),
|
|
["item:a:2"]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn encoder_offer_split_gate() {
|
|
let c = corpus();
|
|
let round = |key: &str| c.run.offer(key).unwrap().err();
|
|
assert_eq!(round("item:a:1"), None);
|
|
assert_eq!(round("item:b:1"), None);
|
|
assert_eq!(round("item:c:1"), Some(OfferRefusal::TestBeforeCheckpoint));
|
|
assert_eq!(round("item:c:2"), Some(OfferRefusal::NoSplit));
|
|
assert_eq!(round("item:a:2"), Some(OfferRefusal::Withdrawn));
|
|
assert!(get(&c, "item:a:2").withdrawn);
|
|
let absent = c.run.offer("item:z:9").err().unwrap().to_string();
|
|
assert!(!absent.contains("item:z:9"), "an absent key is not echoed");
|
|
}
|
|
|
|
#[test]
|
|
fn offers_are_the_live_train_and_validation_items_in_key_order() {
|
|
let c = corpus();
|
|
let keys: Vec<String> = c
|
|
.run
|
|
.offers()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|o| o.key().to_string())
|
|
.collect();
|
|
assert_eq!(keys, ["item:a:1", "item:b:1"]);
|
|
}
|
|
|
|
fn span(slug: &str, start: u32, end: u32) -> NameSpan {
|
|
NameSpan {
|
|
entity: common::eid(slug),
|
|
start,
|
|
end,
|
|
}
|
|
}
|
|
|
|
/// An item naming kay (e1), spoken by hero (e2) after a turn by brokkr (e3),
|
|
/// and a sibling of the same group naming someone (e4).
|
|
fn cited_world() -> (Corpus, CorpusItem, CorpusItem) {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mut entities = entities_in(dir.path());
|
|
entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
|
|
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
|
let group = group_in_bucket(10);
|
|
let mut mine = item_in("item:a:1", "Kay waits here.", &group, cast());
|
|
mine.speaker = common::eid("hero");
|
|
mine.names = vec![span("kay", 0, 3)];
|
|
mine.prior_turns = vec![
|
|
Turn {
|
|
speaker: common::eid("brokkr"),
|
|
text: "Where?".into(),
|
|
},
|
|
Turn {
|
|
speaker: common::eid("brokkr"),
|
|
text: "Again?".into(),
|
|
},
|
|
];
|
|
let mut sibling = item_in("item:a:2", "Someone waits there.", &group, cast());
|
|
sibling.names = vec![span("someone", 0, 7)];
|
|
store.add(mine.clone(), &entities).unwrap();
|
|
store.add(sibling.clone(), &entities).unwrap();
|
|
let run = run_over(dir.path());
|
|
(
|
|
Corpus {
|
|
_dir: dir,
|
|
store,
|
|
run,
|
|
},
|
|
mine,
|
|
sibling,
|
|
)
|
|
}
|
|
|
|
#[test]
|
|
fn encoder_offer_entities_are_the_items_own() {
|
|
let (c, mine, _sibling) = cited_world();
|
|
let t = c.run.train().unwrap();
|
|
assert_eq!(
|
|
mine.cited_entities()
|
|
.map(|e| e.as_str())
|
|
.collect::<Vec<_>>(),
|
|
[
|
|
"entity:kay",
|
|
"entity:hero",
|
|
"entity:brokkr",
|
|
"entity:brokkr"
|
|
],
|
|
"names, speaker, prior-turn speakers, repeats kept"
|
|
);
|
|
let offer = c.run.offer("item:a:1").unwrap().unwrap();
|
|
let ids: Vec<&str> = offer.entities().iter().map(|e| e.as_str()).collect();
|
|
assert_eq!(ids, ["entity:brokkr", "entity:hero", "entity:kay"]);
|
|
assert!(
|
|
!ids.contains(&"entity:someone"),
|
|
"a sibling's entity is not the item's"
|
|
);
|
|
assert_eq!(offer.key(), "item:a:1");
|
|
let inv = c.run.inventory().inventory().unwrap();
|
|
let p = Prompt::encode(&offer, &t, &inv, 0);
|
|
assert!(
|
|
p.text()
|
|
.ends_with("entities: entity:brokkr, entity:hero, entity:kay\n\ninventory rev:0")
|
|
);
|
|
assert!(p.text().contains("[item:a:1]"));
|
|
assert!(!p.text().contains("item:a:2") && !p.text().contains("Someone waits"));
|
|
assert_eq!(p.item_keys(), ["item:a:1"]);
|
|
let sib = c.run.offer("item:a:2").unwrap().unwrap();
|
|
assert_eq!(sib.entities().len(), 2);
|
|
}
|
|
|
|
#[test]
|
|
fn offer_and_store_agree() {
|
|
let mut w = world();
|
|
w.entities.put(ent("kay", "en-US", &["Kay"])).unwrap();
|
|
let mut item = item_in(
|
|
"item:nibu:2",
|
|
"Kay waits here.",
|
|
&group_in_bucket(10),
|
|
cast(),
|
|
);
|
|
item.speaker = common::eid("hero");
|
|
item.names = vec![span("kay", 0, 3)];
|
|
item.prior_turns = vec![Turn {
|
|
speaker: common::eid("brokkr"),
|
|
text: "Where?".into(),
|
|
}];
|
|
w.corpus.add(item.clone(), &w.entities).unwrap();
|
|
let offer = run_over(w.dir.path())
|
|
.offer("item:nibu:2")
|
|
.unwrap()
|
|
.unwrap();
|
|
|
|
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
|
let all = ["nibu", "hero", "kay", "brokkr", "player", "s", "someone"];
|
|
for (n, slug) in all.iter().enumerate() {
|
|
let mut p = common::good();
|
|
p.id = PacketId::parse(&format!("packet:{n:016x}")).unwrap();
|
|
p.provenance.corpus_item = CorpusItemId::parse("item:nibu:2").unwrap();
|
|
p.referents[0].entity = Some(common::eid(slug));
|
|
let offered = offer.entities().contains(&common::eid(slug));
|
|
match packets.admit(p, &w.inventory, &w.corpus, &w.entities) {
|
|
Ok(_) => assert!(offered, "{slug} admitted but not offered"),
|
|
Err(StoreError::Refused(StoreRefusal::EntityNotOnItem { .. })) => {
|
|
assert!(!offered, "{slug} offered but refused")
|
|
}
|
|
Err(other) => panic!("{slug}: {other:?}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Items of one Train group and of a Test group, with the lines each prior
|
|
/// turn quotes.
|
|
#[test]
|
|
fn prior_turns_print_only_text_a_live_train_item_holds() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let entities = entities_in(dir.path());
|
|
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
|
let (train_g, other_train_g) = (group_in_bucket(10), group_in_bucket(11));
|
|
let turn = |text: &str| Turn {
|
|
speaker: common::eid("brokkr"),
|
|
text: text.into(),
|
|
};
|
|
let mut add = |key: &str, text: &str, group: &str, turns: Vec<Turn>| {
|
|
let mut item = item_in(key, text, group, cast());
|
|
item.prior_turns = turns;
|
|
store.add(item, &entities).unwrap();
|
|
};
|
|
add("item:a:1", "withdrawnsecret line", &train_g, vec![]);
|
|
add("item:a:4", "livecontext line", &train_g, vec![]);
|
|
add(
|
|
"item:a:2",
|
|
"reply line",
|
|
&train_g,
|
|
vec![turn("withdrawnsecret line")],
|
|
);
|
|
add(
|
|
"item:a:3",
|
|
"second reply",
|
|
&train_g,
|
|
vec![turn("testsecret line")],
|
|
);
|
|
add(
|
|
"item:a:5",
|
|
"third reply",
|
|
&train_g,
|
|
vec![turn("livecontext line")],
|
|
);
|
|
add(
|
|
"item:a:6",
|
|
"other reply",
|
|
&train_g,
|
|
vec![turn("elsewhere line")],
|
|
);
|
|
add("item:o:1", "elsewhere line", &other_train_g, vec![]);
|
|
add(
|
|
"item:b:1",
|
|
"validation reply",
|
|
&group_in_bucket(50),
|
|
vec![turn("testsecret line")],
|
|
);
|
|
add("item:c:1", "testsecret line", &group_in_bucket(90), vec![]);
|
|
store.withdraw("item:a:1").unwrap();
|
|
|
|
let run = run_over(dir.path());
|
|
let t = run.train().unwrap();
|
|
let inv = run.inventory().inventory().unwrap();
|
|
let mint = Prompt::mint(&t, &inv, 0, &EditFeedback::none());
|
|
for secret in ["withdrawnsecret", "testsecret", "validation reply"] {
|
|
assert!(!mint.text().contains(secret), "mint printed {secret}");
|
|
}
|
|
assert!(
|
|
!mint.text().contains("brokkr"),
|
|
"mint prints no prior turns"
|
|
);
|
|
|
|
let encode = |key: &str| {
|
|
Prompt::encode(&run.offer(key).unwrap().unwrap(), &t, &inv, 0)
|
|
.text()
|
|
.to_string()
|
|
};
|
|
let withdrawn = encode("item:a:2");
|
|
assert!(!withdrawn.contains("withdrawnsecret"));
|
|
assert!(withdrawn.contains("entity:brokkr: [elided]\n[item:a:2]"));
|
|
let test_quote = encode("item:a:3");
|
|
assert!(!test_quote.contains("testsecret"));
|
|
assert!(test_quote.contains("entity:brokkr: [elided]"));
|
|
let validation = encode("item:b:1");
|
|
assert!(!validation.contains("testsecret"));
|
|
assert!(validation.contains("entity:brokkr: [elided]"));
|
|
assert!(encode("item:a:5").contains("entity:brokkr: livecontext line\n[item:a:5]"));
|
|
let cross_group = encode("item:a:6");
|
|
assert!(
|
|
!cross_group.contains("elsewhere"),
|
|
"a held line of another group is not this group's"
|
|
);
|
|
}
|
|
|
|
/// A scratch Train store holding `train_text`, and an encode prompt for a
|
|
/// sibling whose one prior turn is `prior_text`.
|
|
fn encode_with_prior(train_text: &str, prior_text: &str) -> (String, Result<(), StoreError>) {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let entities = entities_in(dir.path());
|
|
let mut store = CorpusStore::open(&dir.path().join("c.cc")).unwrap();
|
|
let g = group_in_bucket(10);
|
|
store
|
|
.add(item_in("item:a:1", train_text, &g, cast()), &entities)
|
|
.unwrap();
|
|
let mut reply = item_in("item:a:2", "reply", &g, cast());
|
|
reply.prior_turns = vec![Turn {
|
|
speaker: common::eid("brokkr"),
|
|
text: prior_text.into(),
|
|
}];
|
|
store.add(reply.clone(), &entities).unwrap();
|
|
let run = run_over(dir.path());
|
|
let t = run.train().unwrap();
|
|
let inv = run.inventory().inventory().unwrap();
|
|
let p = Prompt::encode(&run.offer("item:a:2").unwrap().unwrap(), &t, &inv, 0);
|
|
let text = p.text().to_string();
|
|
// The store's own rule, as the oracle for key equality; after the prompt is
|
|
// built, because an admitted probe would itself become a held line.
|
|
let probe = store.add(item_in("item:a:9", prior_text, &g, cast()), &entities);
|
|
(text, probe.map(|_| ()))
|
|
}
|
|
|
|
#[test]
|
|
fn prior_turn_prints_the_held_train_text() {
|
|
let (text, _) = encode_with_prior("Context Line", "context-line!! :)");
|
|
assert!(
|
|
text.contains("entity:brokkr: Context Line\n[item:a:2]"),
|
|
"{text}"
|
|
);
|
|
for turn_bytes in ["!!", ":)", "context-line"] {
|
|
assert!(!text.contains(turn_bytes), "{turn_bytes} in {text}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn prior_turn_match_uses_the_stores_text_key() {
|
|
let (text, _) = encode_with_prior("Context Line", "context line");
|
|
assert!(text.contains("entity:brokkr: Context Line\n"));
|
|
assert!(!text.contains("context line"));
|
|
}
|
|
|
|
#[test]
|
|
fn prior_turn_fold_matches_the_store() {
|
|
let rows = [
|
|
("case", "Context Line", "context line"),
|
|
(
|
|
"punctuation and spacing",
|
|
"Context Line",
|
|
"context, line!",
|
|
),
|
|
(
|
|
"NFKC fullwidth letters",
|
|
"Context",
|
|
"\u{ff23}\u{ff4f}\u{ff4e}\u{ff54}\u{ff45}\u{ff58}\u{ff54}",
|
|
),
|
|
("NFKC fullwidth digits", "room 12", "room \u{ff11}\u{ff12}"),
|
|
("dotless i", "Ilik", "\u{131}l\u{131}k"),
|
|
("dotted capital I", "Istanbul", "\u{130}stanbul"),
|
|
(
|
|
"katakana vs hiragana",
|
|
"\u{3072}\u{3089}\u{304c}\u{306a}",
|
|
"\u{30d2}\u{30e9}\u{30ac}\u{30ca}",
|
|
),
|
|
("voiced katakana wa", "\u{308f}\u{3099}", "\u{30f7}"),
|
|
("Arabic-Indic digits", "room 12", "room \u{661}\u{662}"),
|
|
];
|
|
for (fold, train_text, prior_text) in rows {
|
|
let (text, probe) = encode_with_prior(train_text, prior_text);
|
|
assert!(
|
|
matches!(
|
|
probe,
|
|
Err(StoreError::Refused(StoreRefusal::S6TextHeld { .. }))
|
|
),
|
|
"{fold}: the store does not hold these as one text: {probe:?}"
|
|
);
|
|
assert!(
|
|
text.contains(&format!("entity:brokkr: {train_text}\n[item:a:2]")),
|
|
"{fold}: {text}"
|
|
);
|
|
assert!(!text.contains(ELIDED_MARK), "{fold}: {text}");
|
|
}
|
|
// A text differing by a letter is a different key: the store admits it and
|
|
// the prior turn is elided.
|
|
let (text, probe) = encode_with_prior("Context Line", "Context Lane");
|
|
assert!(probe.is_ok(), "{probe:?}");
|
|
assert!(
|
|
text.contains("entity:brokkr: [elided]\n[item:a:2]"),
|
|
"{text}"
|
|
);
|
|
}
|
|
|
|
const ELIDED_MARK: &str = "[elided]";
|
|
|
|
#[test]
|
|
fn mint_and_encode_list_the_same_inventory() {
|
|
let w = world();
|
|
let run = run_over(w.dir.path());
|
|
let inv = run.inventory().inventory().unwrap();
|
|
let t = run.train().unwrap();
|
|
let offer = run.offer("item:nibu:1").unwrap().unwrap();
|
|
let fb = EditFeedback::none();
|
|
for rev in [2, 4] {
|
|
let header = format!("inventory rev:{rev}\n");
|
|
let mint = Prompt::mint(&t, &inv, rev, &fb);
|
|
let encode = Prompt::encode(&offer, &t, &inv, rev);
|
|
let from_mint = mint.text().split_once(&header).unwrap().1;
|
|
let from_mint = from_mint.split("\n\n").next().unwrap();
|
|
let from_encode = encode.text().split_once(&header).unwrap().1;
|
|
assert!(!from_mint.is_empty());
|
|
assert_eq!(from_mint, from_encode, "rev {rev}");
|
|
}
|
|
let at = |rev| {
|
|
let text = Prompt::mint(&t, &inv, rev, &fb).text().to_string();
|
|
text.split_once("inventory rev:").unwrap().1.to_string()
|
|
};
|
|
assert_ne!(at(2), at(4), "the revision selects the entries");
|
|
}
|