From 1c08b2abaccde7fcc01b75ae9df715e13ea3b06b Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:18:28 +0000 Subject: [PATCH 1/3] Original corpus (data/original): six passages written for the non-memorised target; recall probes every live item Six short original sentences, model-written (Anthropic, claude-sonnet-5-5) and so labelled, three Train and three Validation, over the Dao revision-3 inventory and one live entity. The recall probe now covers every offered item, naming the work only for a public-domain one, so it can show the original set reads near the guessing floor. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 16 +++--- crates/weksa/src/main.rs | 4 +- crates/weksa/src/run.rs | 29 +++++------ crates/weksa/tests/golf_loop.rs | 22 ++++++--- crates/weksa/tests/original_corpus.rs | 67 ++++++++++++++++++++++++++ data/original/corpus.cc | 1 + data/original/entity.cc | 1 + data/original/inventory.cc | Bin 0 -> 10559 bytes 8 files changed, 108 insertions(+), 32 deletions(-) create mode 100644 crates/weksa/tests/original_corpus.rs create mode 100644 data/original/corpus.cc create mode 100644 data/original/entity.cc create mode 100644 data/original/inventory.cc diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index e45e941..da6262d 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -71,10 +71,10 @@ A referent with an entity is named by one of that entity's forms on the entities Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; /// The instructions of a recall probe (`lowering-recall-diagnostic`): a -/// completion of a public-domain passage from its opening, asked of a model -/// that has no packet and no inventory. It measures recitation, not lowering. +/// completion of a passage from its opening, asked of a model that has no +/// packet and no inventory. It measures recitation, not lowering. pub const RECALL_INSTRUCTIONS: &str = "\ -Below is the opening of a passage from a published public-domain translation. Continue it exactly as that translation words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. +Below is the opening of a passage, with what is known of where it comes from. Continue it exactly as its source words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuation only, not the opening. No code fence, no commentary."; /// The live Train items of a corpus, in key order. The only input minting @@ -547,13 +547,13 @@ impl Prompt { }) } - /// A recall probe: the work, author and translator of a public-domain item - /// and the `opening` of its text, nothing else. Only `RunStores::recall` - /// builds one. - pub(crate) fn recall(key: &str, work: &str, author: &str, translator: &str, opening: &str) -> Self { + /// A recall probe: `provenance` (header lines saying where the item is + /// published, or that it is not) and the `opening` of its text, nothing + /// else. Only `RunStores::recall` builds one. + pub(crate) fn recall(key: &str, provenance: &str, opening: &str) -> Self { Prompt { text: format!( - "{RECALL_INSTRUCTIONS}\n\nwork: {work}\nauthor: {author}\ntranslator: {translator}\n\nopening: {}", + "{RECALL_INSTRUCTIONS}\n\n{provenance}\n\nopening: {}", indented(opening) ), items: vec![key.to_string()], diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 4b65ce6..3808672 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -103,8 +103,8 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, - /// Probe how closely a fresh model recites the run's public-domain items - /// from their openings (chrF of its completion against the item's second + /// Probe how closely a fresh model recites the run's items from their + /// openings (chrF of its completion against the item's second /// half). Prompts and answers live in `/recall`; the command exits 3 /// until every prompt has an answer. A diagnostic, never a gate. Recall { root: PathBuf }, diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 3ae47dd..0d8f1d7 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -803,13 +803,13 @@ pub enum RoundOutcome { } /// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model -/// completes each public-domain item from its opening alone, chrF against the +/// completes each live Train and Validation item from its opening alone, chrF against the /// item's own second half. Recorded nowhere: the answers stay in `/recall`. /// It is never a gate; a high reading says a round's scores on these items are /// recitation. #[derive(Clone, Debug, PartialEq)] pub struct RecallReading { - /// Public-domain items probed, Train and Validation. + /// Items probed, Train and Validation. pub n: u32, /// Items whose completion was a usable answer; a refused one counts 0. pub answered: u32, @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (public-domain items, completion from the opening; a high chrf means round scores on them are recitation)", + "recall n={} answered={} chrf={:.2} (completion from the opening; a high chrf means round scores on these items are recitation; unrelated words score near 0 and an in-voice guess about 20)", self.n, self.answered, self.chrf ) } @@ -843,26 +843,27 @@ fn recall_split(text: &str) -> Option<(String, String)> { } impl RunStores { - /// Probes every live public-domain Train and Validation item of the pinned - /// corpus once, through `model`. + /// Probes every live Train and Validation item of the pinned corpus once, + /// through `model`. A public-domain item names its work, author and + /// translator; any other says `work: not published`. pub fn recall(&self, model: &mut dyn ModelPort) -> Result { let store = |_| RunError::Store; let mut probes = Vec::new(); for item in self.corpus.all().map_err(store)? { - let Source::PublicDomain { - work, - author, - translator, - .. - } = &item.source - else { - continue; + let provenance = match &item.source { + Source::PublicDomain { + work, + author, + translator, + .. + } => format!("work: {work}\nauthor: {author}\ntranslator: {translator}"), + _ => "work: not published".to_string(), }; if EncoderOffer::for_round(&item).is_err() { continue; } if let Some((opening, rest)) = recall_split(&item.text) { - let prompt = Prompt::recall(&item.id, work, author, translator, &opening); + let prompt = Prompt::recall(&item.id, &provenance, &opening); probes.push((prompt, rest)); } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index ae0b7fe..4d9dc1c 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2988,7 +2988,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { } #[test] -fn a_recall_probe_asks_for_the_second_half_of_each_live_public_domain_item_and_never_shows_it() { +fn a_recall_probe_asks_for_the_second_half_of_each_live_item_and_never_shows_it() { use weksa::run::RecallOutcome; let dao = dao(); let mut perfect = reciter(&dao, second_half); @@ -3047,13 +3047,13 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() { } #[test] -fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() { +fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdrawn_item_is_not_probed() { use weksa::corpus::Source; use weksa::run::RecallOutcome; let dao = dao_over(|corpus, entities| { let entities = weksa::EntityStore::open(entities).unwrap(); let mut store = weksa::CorpusStore::open(corpus).unwrap(); - // The same words under other sources: not public domain, not probed. + // Another source: probed too, but it names no work. let mut written = store.get("item:daodejing:2").unwrap().unwrap(); written.id = "item:written:1".into(); written.text = "A line of some other source with words enough to split.".into(); @@ -3074,11 +3074,17 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live let full = both(&dao, None); assert_eq!( (full.n, full.answered), - (2, 2), - "neither the written item nor the withdrawn one is probed" + (3, 3), + "the written item is probed, the withdrawn one is not" ); - // One of two unreadable: the mean falls to a half. + // One of three unreadable: the mean falls to two thirds. let part = both(&dao, Some("Heaven is long-enduring")); - assert_eq!((part.n, part.answered), (2, 1)); - assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf); + assert_eq!((part.n, part.answered), (3, 2)); + assert!((part.chrf - full.chrf * 2.0 / 3.0).abs() < 1e-9, "{}", part.chrf); + // A public-domain prompt names its work; any other says it is not published. + let mut m = reciter(&dao, second_half); + dao.run.recall(&mut m).unwrap(); + let named = m.seen.iter().filter(|t| t.contains("translator: James Legge")).count(); + let unnamed = m.seen.iter().filter(|t| t.contains("work: not published")).count(); + assert_eq!((named, unnamed), (2, 1)); } diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs new file mode 100644 index 0000000..4416a36 --- /dev/null +++ b/crates/weksa/tests/original_corpus.rs @@ -0,0 +1,67 @@ +//! The original corpus (`data/original`): passages written for the +//! non-memorised golf-loop target. It holds no public-domain text, so a model +//! cannot have it in pretraining, and every item says who wrote it. + +use std::path::Path; + +use weksa::corpus::{ModelFamily, Source}; +use weksa::split::{Split, SplitScheme}; +use weksa::{CorpusStore, EntityStore, InventoryStore}; + +fn stores() -> (CorpusStore, EntityStore) { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(dir.join(f), tmp.path().join(f)).unwrap(); + } + let keep = tmp.keep(); + ( + CorpusStore::open(&keep.join("corpus.cc")).unwrap(), + EntityStore::open(&keep.join("entity.cc")).unwrap(), + ) +} + +#[test] +fn every_original_item_records_its_author_and_none_is_public_domain() { + let (corpus, entities) = stores(); + let items = corpus.all().unwrap(); + assert_eq!(items.len(), 6); + for item in &items { + match &item.source { + Source::ModelWritten { + family, generator, .. + } => { + assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); + let generator = generator.as_deref().unwrap_or_default(); + assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); + } + other => panic!("{}: not model-written: {other:?}", item.id), + } + assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id); + } +} + +#[test] +fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { + let (corpus, _) = stores(); + let splits: Vec> = corpus + .all() + .unwrap() + .iter() + .map(|i| SplitScheme::V1.live(i)) + .collect(); + let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); + assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3)); + assert_eq!(count(Split::Test), 0); +} + +#[test] +fn the_original_inventory_is_the_dao_revision_three() { + let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); + let own = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original/inventory.cc"); + assert_eq!(std::fs::read(dao).unwrap(), std::fs::read(&own).unwrap()); + let tmp = tempfile::tempdir().unwrap(); + let copy = tmp.path().join("inventory.cc"); + std::fs::copy(&own, ©).unwrap(); + assert!(InventoryStore::open(©).is_ok()); +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc new file mode 100644 index 0000000..c11443f --- /dev/null +++ b/data/original/corpus.cc @@ -0,0 +1 @@ +cultcache.store.v1weksa.corpus_itemweksa.corpus_itemweksa.corpus_item.v1weksa.corpus_itemV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}weksa.corpus_itemitem:original:1weksa.corpus_item2026-11-10T14:17:08Z훯item:original:10A quiet mind is like still water in a deep bowl.en-USentity:keeperoriginal-bowlModelWrittenAnthropic]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)2026-11-10item:original:2weksa.corpus_item2026-11-10T14:17:08Zitem:original:2IThe river weakens the bank in spring and strengthens the roots in autumn.en-USentity:keeperoriginal-riverModelWrittenAnthropic]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)2026-11-10item:original:3weksa.corpus_item2026-11-10T14:17:08Zitem:original:37The sage waits at the door until the water has settled.en-USentity:keeperoriginal-doorModelWrittenAnthropic]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)2026-11-10item:original:4weksa.corpus_item2026-11-10T14:17:08Zitem:original:4CThe old gardener empties the jar and fills it again with rainwater.en-USentity:keeperoriginal-jarModelWrittenAnthropic]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)2026-11-10item:original:5weksa.corpus_item2026-11-10T14:17:08Zitem:original:5JUnder the same sky the earth keeps its patience, and the baker keeps hers.en-USentity:keeperoriginal-skyModelWrittenAnthropic]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)2026-11-10item:original:6weksa.corpus_item2026-11-10T14:17:08Z雯item:original:6+A hungry belly argues louder than any will.en-USentity:keeperoriginal-bellyModelWrittenAnthropic]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)2026-11-10 \ No newline at end of file diff --git a/data/original/entity.cc b/data/original/entity.cc new file mode 100644 index 0000000..c92e33d --- /dev/null +++ b/data/original/entity.cc @@ -0,0 +1 @@ +cultcache.store.v1weksa.entityweksa.entityweksa.entity.v1weksa.entityL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}weksa.entityentity:keeperweksa.entity2026-11-10T14:17:08Z:entity:keeperen-USthe keeperLiveweksa-original \ No newline at end of file diff --git a/data/original/inventory.cc b/data/original/inventory.cc new file mode 100644 index 0000000000000000000000000000000000000000..373eb3cf259663321fdf94128b4f11a984998f01 GIT binary patch literal 10559 zcmb7KTaP1072eA)2xMn=ClfY1T%O2g+`i682q7Wz21r3XfS@UNm+fZS-KMHNUW0hZ z3(vdD#1YTy_GM;CmMBO_NN5!$KO)lp7R2eU>Z~b4JfNc^E|JGy><`g@0^b@BLTpvj9{7_}9Pv(-DW>#^k4D zOpi{FYMUQ@Uq19p%6Je*yPkt?;L$N1_u;vt(_jDQ&rj;Ref)8SC!^;e?uQLXdIt5 zI`H?aFYhjNyA6$e5)CPP`S{k$i%W%fTtB__fOi%~!{sap!V8yk63lc zL5cG{V`5wX;S=Ifw@;H9rI9OS;F8F9YckkDiT1A1PkY^mu*EZbN>GY50Xu6i9>av# zX@HXKJ%eF)!zhP!I24FV1E`?wX$>@2laS2i`lHZLfie`;g5kmzCWgSna;D%=@wlhS zwYPi}Ox$r0`DH*;0X)G4n?SPVG$|!D+^23F0x0%EtVL>wgaQ(AM0y{bW6kI$=f^)iw=#XR|o?QK$ zQ6Ds)J#Z5?AF5kaHHC7b8!q7^OW%iTLZ7TYNC;HZ5;Cfme#by=tU2)`%K0_msLJ|| z0q#8eeLTMg7^PH`b#v_pd^qLFHQ*@We%D~#+VClxuz=HR&`~-z(c7D>E@ZpEYtT`m z{+_|Tvn7K8^|H*n1|B8yBLlvh@)4Y&u7O5r-elg}LN!kq38L!6C;_4}@qH~^CrxO? zsZ#C5azc2L?Rhxtc~*mIRu(o%Fs30$!4|OU9+j`Z)7>{$gijzUs{fBJCzR!JI;YQr zu{N$IEFJ z%5Ed-0W-qpYoGD}YTP#NhXE8l>hGk6(#Z#!TKn&CR`vx0li$V4hn~G;6GQ{bfCFp- zCNTpFYr^?-d<_+psE#$2PCBCgP$vD;?T-WQ#Z$(qRpwQepw!jU{dBh;Wl1IgSuia-zcV~n} z`N;}4Sn@kmfjB4-o#!y=lx621D4hBJO5&jue^lT#KbLzecNZ>7v2PT(t!E(4{D>my zq12^@+s=l|NF??xCLT&6dj+k|CJf-Z0BVDYg_6Pj0;~Iv?BK*JCnq^zRKCAmfc8Ee zlR2E2TzLY4Ei~*cKV=k32*lhygIzT?8SU09DAGJj9@v+?u#i#vzQaYOc`A?QHPEPy z-~9@7V?+Acs+*-1pj(4Nsp+kS-8vnzcsi*AM#<`1HgIzjvmuEBKCZ(?NvlG-mBJw~ zyQ-+p21;xd((Mi0F9m}gGmR471A9U`S3$%R*ei(gRi%#7;@dWScTK%GjE8WFszRd_ zS()x$3MrD*PB&`JIkY*Stl)|s-Xr9cFBuK<(->CdM#C#Fma3N7aiu{~`aW2gEjO?S z&DI?#oiMk?Hz;B6nV5}bJc}xLC@q)p9KfR$Jd}j@O@7VvEXwbd$^%B}wglKpBiKz; z7^3ugXfkX=+~wJzf{Ie=J0@x;zqw^_@Zjz&$nKh90(6F<0t6RpD*fQ@7D|g74=g}b zgbobA=2pDkszRb-1b3{8W9u@E;X;J^E)gq@{t6W)cw~a7nH3x+xjtm(-;oqpGDzueb8Zl*=U7y`YMunKUPB8WA|uiTqx{JtcBG z{*8rfl)}AV_TB)-DLkk|epA9^Nt^^_AXEb1l|aoa`UtjS00pNhgB2=u_a#{CGFw;~ z94kvBM5XLEC02XI;n*k_4>41yWPL;8be6+7d!3->8cifD5O@LH4CJ()!(|8cMlS4ia0~9eMX4#=ns7J~S1?ftwpYS*o)Lal0YoL*K?$&_ zuhC%|P5P*WD}%P+W-1=T1!RROD)aVBOxw%syj(#<<>1{CVh1+(lvd+|O2{%~H%&$) zszwNvnTI8oy)A)T`*>JPX9^px)7`(OsQq2RSGWIrar5P?8$bB)*^hkRI_EU%oKBsy zS?6rkIooy4PMxz`=j_!vPjYAFa9QgWW76x4ZHwXQ@5RWpEe54+F&=G;p=eu-Lfc{h z+7{f6?eTF=+i(bmhC@&^9D<|a5F`zUU}-o6O}WtJJ<27{_$*+BUm%Ll0!(~1al}xC zUkuZ%8e$lxM~uVth#pOk=-2d!-c66-V0r`-(B16P Date: Tue, 10 Nov 2026 14:21:31 +0000 Subject: [PATCH 2/3] Encode prompt states the id form; recall line states the real chrF floor; probe test over data/original Five of six encoders in the first original-corpus round wrote bare integer ids and were refused at parse (encoder-prompt-concept-ids). The prompt now says ids are written in full with their family prefix. Soul s1 on the original corpus: unrelated English scores chrF 13 to 18, not near 0, so the recall line says so; a test runs the probe over the real original items. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/context.rs | 18 ++++++++++ crates/weksa/tests/original_corpus.rs | 47 +++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index da6262d..3c73b0c 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -59,7 +59,7 @@ Fields: - predications: a list of {id, concept, frame, roles, features}; roles is a list of {role, filler}, filler a node id. - relations: a list of {relation, from, to, features}. - stance: {features}; the speaker's felt affect and restraint, never the affect the line displays. -Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. An entity comes only from the entities line. +Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": label} or {\"Flag\": true or false}, as the inventory lists the feature's kind."; /// The instructions a lowering prompt starts with: the lowerer's whole brief. diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 0d8f1d7..77c3642 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (completion from the opening; a high chrf means round scores on these items are recitation; unrelated words score near 0 and an in-voice guess about 20)", + "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 13 to 18, so a reading near that floor is no recitation and one well above it is)", self.n, self.answered, self.chrf ) } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index fe351d7..04ebe35 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -686,3 +686,21 @@ mod lowering { } } } + +/// First real rounds: five of six encoders wrote bare integers (`"concept": 7`) +/// where the inventory lists `concept:7`, and the draft reader refused them. +/// The prompt says the form outright, and the inventory lines it points to +/// are spelled in that form. +#[test] +fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { + assert!(ENCODE.contains("family prefix included: concept:7, never 7")); + let (text, _) = encode_with_prior("Hello there.", "Hi."); + let listed = text + .lines() + .skip_while(|l| !l.starts_with("inventory rev:")) + .skip(1); + for line in listed { + let id = line.split(": ").next().unwrap(); + assert!(id.contains(':'), "an inventory line spells its id in full: {line}"); + } +} diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 4416a36..51e4c35 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -65,3 +65,50 @@ fn the_original_inventory_is_the_dao_revision_three() { std::fs::copy(&own, ©).unwrap(); assert!(InventoryStore::open(©).is_ok()); } + +/// The probe over the real original corpus: every item is offered, none names a +/// work, and a model that answers with unrelated words reads at the noise floor. +#[test] +fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { + use weksa::model::{ModelPort, PortError}; + use weksa::run::{RecallOutcome, RunPaths, RunStores}; + struct Unrelated(Vec); + impl ModelPort for Unrelated { + fn model(&self) -> &str { + "unrelated" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"stone harbours remember distant lanterns"}"#.to_string()) + } + } + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + let path = |f: &str| { + std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); + tmp.path().join(f) + }; + let (c, i, e) = (path("corpus.cc"), path("inventory.cc"), path("entity.cc")); + let run = RunStores::create( + &tmp.path().join("run"), + "orig", + RunPaths { + corpus: &c, + inventory: &i, + entities: &e, + }, + "m", + "bag", + weksa::run::LowererReach::BriefedSubagent, + ) + .unwrap(); + let mut model = Unrelated(vec![]); + let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered), (6, 6)); + assert!(r.chrf < 25.0, "{}", r.chrf); + assert_eq!(model.0.len(), 6); + assert!(model.0.iter().all(|t| t.contains("work: not published"))); + assert!(model.0.iter().all(|t| !t.contains("translator:"))); +} -- 2.47.3 From 5ebdb3cff44512a14490b0198228e442a6a6abd3 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:24:15 +0000 Subject: [PATCH 3/3] Soul s2 lows: the id-form test lists a real inventory; the floor reads 14 to 20 Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/context.rs | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 77c3642..bcf0d00 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 13 to 18, so a reading near that floor is no recitation and one well above it is)", + "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 14 to 20, so a reading near that floor is no recitation and one well above it is)", self.n, self.answered, self.chrf ) } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 04ebe35..c379f0e 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -694,11 +694,22 @@ mod lowering { #[test] fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { assert!(ENCODE.contains("family prefix included: concept:7, never 7")); - let (text, _) = encode_with_prior("Hello there.", "Hi."); - let listed = text + let w = world(); + let run = run_over(w.dir.path()); + let inv = run.inventory().inventory().unwrap(); + let t = run.train().unwrap(); + let offer = run.offer("item:nibu:1").unwrap().unwrap(); + let text = Prompt::encode(&offer, &t, &inv, 4).text().to_string(); + let listed: Vec<&str> = text + .split_once("inventory rev:4\n") + .unwrap() + .1 + .split("\n\n") + .next() + .unwrap() .lines() - .skip_while(|l| !l.starts_with("inventory rev:")) - .skip(1); + .collect(); + assert!(!listed.is_empty(), "the prompt lists entries"); for line in listed { let id = line.split(": ").next().unwrap(); assert!(id.contains(':'), "an inventory line spells its id in full: {line}"); -- 2.47.3