From fc52f3d3a49bb312fc42f46a869cd1c233b7ae5c Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 05:01:18 +0000 Subject: [PATCH] RoundRecord keys 16 inventory_digest and 17 cycle; round becomes the drift check, score_round and one put An encode answer the port reports as InvalidData is that item's Refused(Parse), not an aborted round. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 83 +++++++++++++++++--- crates/weksa/tests/golf_loop.rs | 30 +++++++ schemas/cultnet/weksa.loop_round.schema.json | 18 ++++- 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 46670c8..a8ea6d6 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -147,6 +147,8 @@ pub struct RefusalCounts { /// validation_mean: SplitMean::default(), /// validation_scored: 0, /// validation_refused: RefusalCounts::default(), +/// inventory_digest: String::new(), +/// cycle: None, /// } /// } /// ``` @@ -190,6 +192,12 @@ pub struct RoundRecord { /// raise it. #[cultcache(key = 15)] validation_scored: u32, + /// 16 hex of the inventory file the round was scored under. + #[cultcache(key = 16)] + inventory_digest: String, + /// `Some(k)` for the candidate round of cycle `k`. + #[cultcache(key = 17)] + cycle: Option, } impl RoundRecord { @@ -241,6 +249,12 @@ impl RoundRecord { pub fn validation_scored(&self) -> u32 { self.validation_scored } + pub fn inventory_digest(&self) -> &str { + &self.inventory_digest + } + pub fn cycle(&self) -> Option { + self.cycle + } /// The one line a recorded round prints. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -564,14 +578,53 @@ pub fn round( model: &mut M, scorer: &Scorer, ) -> Result { + check_drift(run, model, scorer)?; let store = |_| RunError::Store; + let dir = run.next_round_dir().map_err(store)?; + let digest = run.record.inventory_digest.clone(); + match score_round(run, &run.inventory, &digest, &dir, None, model, scorer)? { + RoundOutcome::Recorded(record) => { + run.cache + .put( + format!("run:{}:round:{}", run.record.label, record.n), + &record, + ) + .map_err(store)?; + Ok(RoundOutcome::Recorded(record)) + } + unanswered => Ok(unanswered), + } +} + +/// The model and embedder the run pinned are the ones answering. +fn check_drift( + run: &RunStores, + model: &M, + scorer: &Scorer, +) -> Result<(), RunError> { if model.model() != run.record.model { return Err(RunError::Drift(DriftOf::Model)); } if scorer.embedder().model() != run.record.embedder { return Err(RunError::Drift(DriftOf::Embedder)); } - let inventory = run.inventory.inventory().map_err(store)?; + Ok(()) +} + +/// Encodes, admits, lowers and scores every offered item under `inventory` +/// (whose file digests to `inventory_digest`), with packets in `dir`, and +/// returns the record unput. `Recorded` here means built, not stored. +fn score_round( + run: &RunStores, + inventory_store: &InventoryStore, + inventory_digest: &str, + dir: &Path, + cycle: Option, + model: &mut M, + scorer: &Scorer, +) -> Result { + let store = |_| RunError::Store; + let inventory = inventory_store.inventory().map_err(store)?; let rev = inventory.latest().ok_or(RunError::NoInventory)?.n; let train = run.train().map_err(store)?; @@ -581,7 +634,11 @@ pub fn round( for offer in run.offers().map_err(store)? { let prompt = Prompt::encode(&offer, &train, &inventory, rev); match model.answer(&prompt) { - Ok(answer) => answers.push((offer.key().to_string(), answer)), + Ok(answer) => answers.push((offer.key().to_string(), Some(answer))), + // An answer that is not UTF-8 is that item's parse refusal. + Err(PortError::Io(ErrorKind::InvalidData)) => { + answers.push((offer.key().to_string(), None)) + } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } @@ -596,10 +653,9 @@ pub fn round( }); } - let dir = run.next_round_dir().map_err(store)?; - std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + std::fs::create_dir_all(dir).map_err(|e| RunError::Io(e.kind()))?; // The directory of an unrecorded round holds only a failed attempt's packets. - discard_packets(&dir); + discard_packets(dir); let mut packets = PacketStore::open(&dir.join("packets.cc")).map_err(store)?; let encoder = encoder_id(model.model(), &run.record.prompt_digest); let mut items: Vec = Vec::new(); @@ -607,12 +663,12 @@ pub fn round( for (key, answer) in &answers { let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?; let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?; - let fate = match PacketDraft::parse(answer) { - Err(_) => Fate::Refused(Stage::Parse, None), - Ok(draft) => { + let fate = match answer.as_deref().map(PacketDraft::parse) { + None | Some(Err(_)) => Fate::Refused(Stage::Parse, None), + Some(Ok(draft)) => { let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); - match packets.admit(packet, &run.inventory, &run.corpus, &run.entities) { + match packets.admit(packet, inventory_store, &run.corpus, &run.entities) { Err(_) => Fate::Refused(Stage::Admit, None), Ok(view) => match lower(&view, &inventory, &run.entities) { Err(_) => Fate::Refused(Stage::Lower, Some(packet_key)), @@ -647,7 +703,7 @@ pub fn round( match scorer.score(&inputs) { Ok(scores) => scores, Err(_) => { - discard_packets(&dir); + discard_packets(dir); return Err(RunError::Embed); } } @@ -720,10 +776,9 @@ pub fn round( validation_refused: refused, train_scored, validation_scored, + inventory_digest: inventory_digest.to_string(), + cycle, }; - run.cache - .put(format!("run:{}:round:{n}", run.record.label), &record) - .map_err(store)?; Ok(RoundOutcome::Recorded(record)) } @@ -759,6 +814,8 @@ mod tests { validation_mean: SplitMean::default(), validation_scored: 0, validation_refused: RefusalCounts::default(), + inventory_digest: String::new(), + cycle: None, }; assert_eq!(r.n(), 1); } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index c2eec2b..71421ff 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -624,6 +624,8 @@ fn round_over_the_dao_stores_scores_train_and_validation() { assert_eq!(r.model(), "fake-model"); assert_eq!(r.embedder(), "bag"); assert_eq!(r.chrf_version(), weksa::score::CHRF_VERSION); + assert_eq!(r.inventory_digest(), pinned.inventory_digest); + assert_eq!(r.cycle(), None, "a plain round belongs to no cycle"); // It is stored, and the packets are in the round's own store, made by the encoder step. assert_eq!( @@ -946,6 +948,34 @@ fn refusal_stages_are_counted() { assert_eq!(full.validation_scored(), 2); } +#[test] +fn non_utf8_encode_answer_is_a_parse_refusal() { + let mut dao = dao(); + let expected = prompts(&dao.run); + let answers: HashMap = good_answers() + .into_iter() + .map(|(n, a)| (format!("item:daodejing:{n}"), a)) + .collect(); + let round1 = dao.root.join("round-1"); + std::fs::create_dir_all(&round1).unwrap(); + for (key, p) in &expected { + let bytes = if key == "item:daodejing:3" { + vec![0xff, 0xfe, 0xfd] + } else { + answers[key].clone().into_bytes() + }; + std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap(); + } + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); + // Item 3 is Validation: its refusal is a Parse refusal and the round stands. + let v = r.validation_refused(); + assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0)); + assert_eq!((r.train_scored(), r.validation_scored()), (1, 1)); + assert_eq!(r.validation_mean().count, 2); + assert_eq!(dao.run.rounds().unwrap(), [r]); +} + /// An embedder that is down. struct Down; diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index 7cde9c2..d846e12 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n }\n}\n```", "type": "object", "properties": { "calls": { @@ -16,12 +16,25 @@ "corpus_digest": { "type": "string" }, + "cycle": { + "description": "`Some(k)` for the candidate round of cycle `k`.", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, "embedder": { "type": "string" }, "entity_digest": { "type": "string" }, + "inventory_digest": { + "description": "16 hex of the inventory file the round was scored under.", + "type": "string" + }, "inventory_rev": { "type": "integer", "format": "uint32", @@ -85,7 +98,8 @@ "validation_mean", "validation_refused", "train_scored", - "validation_scored" + "validation_scored", + "inventory_digest" ], "$defs": { "Call": {