From a14bcaf9f830ef4921dc8316823e270a929ea6c8 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 05:00:34 +0000 Subject: [PATCH 01/19] The run stops opening the caller's inventory store: create copies it to /inventory.cc and pins its digest (key 11) RunRecord.inventory is provenance only; open refuses Drift(Inventory) when the run's own file differs. A run.cc from before has no key 11 and its open fails. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 33 ++++- crates/weksa/tests/golf_loop.rs | 144 +++++++++++++++++++++ schemas/cultnet/weksa.loop_run.schema.json | 8 +- 3 files changed, 181 insertions(+), 4 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index de6d846..46670c8 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2,7 +2,11 @@ //! and the one writer of both (`RunStores::create` and, for rounds, `round`). //! //! A run pins its corpus and entity store files by canonical path and FNV-1a 64 -//! digest of their bytes. Every round reopens them only from that record and +//! digest of their bytes. It works on its own copy of the inventory: +//! `RunStores::create` copies the caller's inventory file to +//! `/inventory.cc` and pins the digest of the copy; the caller's path is +//! kept as provenance and never opened again, and `open` refuses a head file +//! whose digest differs. Every round reopens them only from that record and //! refuses on drift, and every round record carries the digests. Nothing inside //! a process whose writer controls the code can stop an item faked into a //! scratch store; what this makes true is that a run over a scratch store is a @@ -46,6 +50,9 @@ pub struct RunRecord { /// Canonical absolute path of the corpus store. #[cultcache(key = 1)] pub corpus: String, + /// Canonical absolute path of the inventory store the run was opened over: + /// provenance only. `create` copies that file to `/inventory.cc` and + /// the run reads nothing at this path afterwards. #[cultcache(key = 2)] pub inventory: String, #[cultcache(key = 3)] @@ -67,6 +74,9 @@ pub struct RunRecord { pub embedder: String, #[cultcache(key = 10)] pub chrf_version: String, + /// 16 hex of the bytes `create` copied to `/inventory.cc`. + #[cultcache(key = 11)] + pub inventory_digest: String, } /// Where a round refused: the first stage that did. @@ -276,6 +286,8 @@ pub enum DriftOf { Lowerer, Model, Embedder, + /// The run's own head inventory file. + Inventory, } /// A run's failure. Display and Debug name a kind, never a path, key, label or @@ -340,10 +352,17 @@ pub struct RunStores { entities: EntityStore, } -fn file_digest(path: &str) -> Option { +/// The run's own copy of the inventory, made at `create`. +const INVENTORY_FILE: &str = "inventory.cc"; + +fn file_digest_at(path: &Path) -> Option { std::fs::read(path).ok().map(|b| hex16(fnv1a64(&b))) } +fn file_digest(path: &str) -> Option { + file_digest_at(Path::new(path)) +} + fn canonical(path: &Path) -> Result { std::fs::canonicalize(path) .map_err(|e| RunError::Io(e.kind()))? @@ -403,10 +422,15 @@ impl RunStores { model: model.to_string(), embedder: embedder.to_string(), chrf_version: CHRF_VERSION.to_string(), + inventory_digest: String::new(), }; + // The run works on its own copy; the caller's store is never opened again. + let head = root.join(INVENTORY_FILE); + std::fs::copy(&record.inventory, &head).map_err(|e| RunError::Io(e.kind()))?; let record = RunRecord { corpus_digest: file_digest(&record.corpus).ok_or(RunError::Store)?, entity_digest: file_digest(&record.entities).ok_or(RunError::Store)?, + inventory_digest: file_digest_at(&head).ok_or(RunError::Store)?, ..record }; cache @@ -435,6 +459,9 @@ impl RunStores { if file_digest(&record.entities).as_ref() != Some(&record.entity_digest) { return Err(RunError::Drift(DriftOf::Entities)); } + if file_digest_at(&root.join(INVENTORY_FILE)).as_ref() != Some(&record.inventory_digest) { + return Err(RunError::Drift(DriftOf::Inventory)); + } Self::with(root, cache, record) } @@ -443,7 +470,7 @@ impl RunStores { Ok(Self { root: root.to_path_buf(), corpus: CorpusStore::open(Path::new(&record.corpus)).map_err(store)?, - inventory: InventoryStore::open(Path::new(&record.inventory)).map_err(store)?, + inventory: InventoryStore::open(&root.join(INVENTORY_FILE)).map_err(store)?, entities: EntityStore::open(Path::new(&record.entities)).map_err(store)?, cache, record, diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 6631861..c2eec2b 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -120,6 +120,10 @@ fn open_checks_each_pin_by_name() { drift(&|r| r.entity_digest = "0".repeat(16)), Some(RunError::Drift(DriftOf::Entities)) ); + assert_eq!( + drift(&|r| r.inventory_digest = "0".repeat(16)), + Some(RunError::Drift(DriftOf::Inventory)) + ); assert_eq!(drift(&|_| {}), None, "the unchanged record opens"); // An entity put after the pin is another entity store. @@ -132,6 +136,146 @@ fn open_checks_each_pin_by_name() { ); } +#[test] +fn a_run_works_on_its_own_copy_of_the_inventory() { + let w = world(); + let dir = w.dir.path().to_path_buf(); + let root = dir.join("run"); + let (c, i, e) = paths(&dir); + let source_bytes = std::fs::read(&i).unwrap(); + let mut run = RunStores::create( + &root, + "t", + RunPaths { + corpus: &c, + inventory: &i, + entities: &e, + }, + "fake-model", + "bag", + ) + .unwrap(); + + // The copy is byte for byte the source, and the record pins its digest. + let copy = root.join("inventory.cc"); + assert_eq!(std::fs::read(©).unwrap(), source_bytes); + assert_eq!(run.record().inventory_digest, hex(&i)); + assert_eq!(run.record().inventory_digest, hex(©)); + assert_eq!( + run.record().inventory, + std::fs::canonicalize(&i).unwrap().to_str().unwrap(), + "the source path is provenance" + ); + let rev = run.inventory().inventory().unwrap().latest().unwrap().n; + + // A revision written to the source after open is a revision the run never + // reads, and it does not drift the run. + let mut source = weksa::InventoryStore::open(&i).unwrap(); + source + .admit_revision( + weksa::InventoryRevision { + n: rev + 1, + parent: Some(rev), + changes: vec![], + frozen: false, + }, + vec![], + ) + .unwrap(); + assert_ne!(hex(&i), run.record().inventory_digest); + run = RunStores::open(&root).unwrap(); + assert_eq!( + run.inventory().inventory().unwrap().latest().unwrap().n, + rev, + "the run reads its copy" + ); + assert_eq!(std::fs::read(©).unwrap(), source_bytes); + + // Even with the source gone the run opens. + std::fs::remove_file(&i).unwrap(); + assert!(RunStores::open(&root).is_ok()); + + // A revision appended to the copy is drift. + let mut head = weksa::InventoryStore::open(©).unwrap(); + head.admit_revision( + weksa::InventoryRevision { + n: rev + 1, + parent: Some(rev), + changes: vec![], + frozen: false, + }, + vec![], + ) + .unwrap(); + assert_eq!( + RunStores::open(&root).err(), + Some(RunError::Drift(DriftOf::Inventory)) + ); +} + +/// `weksa.loop_run` as it was before the run kept its own inventory copy: keys +/// 0 to 10. +#[derive(Clone, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_run")] +struct LegacyRun { + #[cultcache(key = 0)] + label: String, + #[cultcache(key = 1)] + corpus: String, + #[cultcache(key = 2)] + inventory: String, + #[cultcache(key = 3)] + entities: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + prompt_digest: String, + #[cultcache(key = 7)] + lowerer_digest: String, + #[cultcache(key = 8)] + model: String, + #[cultcache(key = 9)] + embedder: String, + #[cultcache(key = 10)] + chrf_version: String, +} + +#[test] +fn a_run_from_before_the_copy_does_not_open() { + // A run.cc without key 11 is a scratch run off git: its open fails Store. + let w = world(); + let dir = w.dir.path().to_path_buf(); + let current = run_over(&dir).record().clone(); + let root = dir.join("run"); + std::fs::remove_file(root.join("run.cc")).unwrap(); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache + .put( + "run:t", + &LegacyRun { + label: current.label, + corpus: current.corpus, + inventory: current.inventory, + entities: current.entities, + corpus_digest: current.corpus_digest, + entity_digest: current.entity_digest, + prompt_digest: current.prompt_digest, + lowerer_digest: current.lowerer_digest, + model: current.model, + embedder: current.embedder, + chrf_version: current.chrf_version, + }, + ) + .unwrap(); + assert_eq!(RunStores::open(&root).err(), Some(RunError::Store)); +} + #[test] fn source_digests_cover_their_files() { let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); diff --git a/schemas/cultnet/weksa.loop_run.schema.json b/schemas/cultnet/weksa.loop_run.schema.json index b04b8b4..69bb0a4 100644 --- a/schemas/cultnet/weksa.loop_run.schema.json +++ b/schemas/cultnet/weksa.loop_run.schema.json @@ -26,6 +26,11 @@ "type": "string" }, "inventory": { + "description": "Canonical absolute path of the inventory store the run was opened over:\nprovenance only. `create` copies that file to `/inventory.cc` and\nthe run reads nothing at this path afterwards.", + "type": "string" + }, + "inventory_digest": { + "description": "16 hex of the bytes `create` copied to `/inventory.cc`.", "type": "string" }, "label": { @@ -53,6 +58,7 @@ "lowerer_digest", "model", "embedder", - "chrf_version" + "chrf_version", + "inventory_digest" ] } -- 2.47.3 From fc52f3d3a49bb312fc42f46a869cd1c233b7ae5c Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 05:01:18 +0000 Subject: [PATCH 02/19] RoundRecord keys 16 inventory_digest and 17 cycle; round becomes the drift check, score_round and one put An encode answer the port reports as InvalidData is that item's Refused(Parse), not an aborted round. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 83 +++++++++++++++++--- crates/weksa/tests/golf_loop.rs | 30 +++++++ schemas/cultnet/weksa.loop_round.schema.json | 18 ++++- 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 46670c8..a8ea6d6 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -147,6 +147,8 @@ pub struct RefusalCounts { /// validation_mean: SplitMean::default(), /// validation_scored: 0, /// validation_refused: RefusalCounts::default(), +/// inventory_digest: String::new(), +/// cycle: None, /// } /// } /// ``` @@ -190,6 +192,12 @@ pub struct RoundRecord { /// raise it. #[cultcache(key = 15)] validation_scored: u32, + /// 16 hex of the inventory file the round was scored under. + #[cultcache(key = 16)] + inventory_digest: String, + /// `Some(k)` for the candidate round of cycle `k`. + #[cultcache(key = 17)] + cycle: Option, } impl RoundRecord { @@ -241,6 +249,12 @@ impl RoundRecord { pub fn validation_scored(&self) -> u32 { self.validation_scored } + pub fn inventory_digest(&self) -> &str { + &self.inventory_digest + } + pub fn cycle(&self) -> Option { + self.cycle + } /// The one line a recorded round prints. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -564,14 +578,53 @@ pub fn round( model: &mut M, scorer: &Scorer, ) -> Result { + check_drift(run, model, scorer)?; let store = |_| RunError::Store; + let dir = run.next_round_dir().map_err(store)?; + let digest = run.record.inventory_digest.clone(); + match score_round(run, &run.inventory, &digest, &dir, None, model, scorer)? { + RoundOutcome::Recorded(record) => { + run.cache + .put( + format!("run:{}:round:{}", run.record.label, record.n), + &record, + ) + .map_err(store)?; + Ok(RoundOutcome::Recorded(record)) + } + unanswered => Ok(unanswered), + } +} + +/// The model and embedder the run pinned are the ones answering. +fn check_drift( + run: &RunStores, + model: &M, + scorer: &Scorer, +) -> Result<(), RunError> { if model.model() != run.record.model { return Err(RunError::Drift(DriftOf::Model)); } if scorer.embedder().model() != run.record.embedder { return Err(RunError::Drift(DriftOf::Embedder)); } - let inventory = run.inventory.inventory().map_err(store)?; + Ok(()) +} + +/// Encodes, admits, lowers and scores every offered item under `inventory` +/// (whose file digests to `inventory_digest`), with packets in `dir`, and +/// returns the record unput. `Recorded` here means built, not stored. +fn score_round( + run: &RunStores, + inventory_store: &InventoryStore, + inventory_digest: &str, + dir: &Path, + cycle: Option, + model: &mut M, + scorer: &Scorer, +) -> Result { + let store = |_| RunError::Store; + let inventory = inventory_store.inventory().map_err(store)?; let rev = inventory.latest().ok_or(RunError::NoInventory)?.n; let train = run.train().map_err(store)?; @@ -581,7 +634,11 @@ pub fn round( for offer in run.offers().map_err(store)? { let prompt = Prompt::encode(&offer, &train, &inventory, rev); match model.answer(&prompt) { - Ok(answer) => answers.push((offer.key().to_string(), answer)), + Ok(answer) => answers.push((offer.key().to_string(), Some(answer))), + // An answer that is not UTF-8 is that item's parse refusal. + Err(PortError::Io(ErrorKind::InvalidData)) => { + answers.push((offer.key().to_string(), None)) + } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } @@ -596,10 +653,9 @@ pub fn round( }); } - let dir = run.next_round_dir().map_err(store)?; - std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?; + std::fs::create_dir_all(dir).map_err(|e| RunError::Io(e.kind()))?; // The directory of an unrecorded round holds only a failed attempt's packets. - discard_packets(&dir); + discard_packets(dir); let mut packets = PacketStore::open(&dir.join("packets.cc")).map_err(store)?; let encoder = encoder_id(model.model(), &run.record.prompt_digest); let mut items: Vec = Vec::new(); @@ -607,12 +663,12 @@ pub fn round( for (key, answer) in &answers { let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?; let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?; - let fate = match PacketDraft::parse(answer) { - Err(_) => Fate::Refused(Stage::Parse, None), - Ok(draft) => { + let fate = match answer.as_deref().map(PacketDraft::parse) { + None | Some(Err(_)) => Fate::Refused(Stage::Parse, None), + Some(Ok(draft)) => { let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); - match packets.admit(packet, &run.inventory, &run.corpus, &run.entities) { + match packets.admit(packet, inventory_store, &run.corpus, &run.entities) { Err(_) => Fate::Refused(Stage::Admit, None), Ok(view) => match lower(&view, &inventory, &run.entities) { Err(_) => Fate::Refused(Stage::Lower, Some(packet_key)), @@ -647,7 +703,7 @@ pub fn round( match scorer.score(&inputs) { Ok(scores) => scores, Err(_) => { - discard_packets(&dir); + discard_packets(dir); return Err(RunError::Embed); } } @@ -720,10 +776,9 @@ pub fn round( validation_refused: refused, train_scored, validation_scored, + inventory_digest: inventory_digest.to_string(), + cycle, }; - run.cache - .put(format!("run:{}:round:{n}", run.record.label), &record) - .map_err(store)?; Ok(RoundOutcome::Recorded(record)) } @@ -759,6 +814,8 @@ mod tests { validation_mean: SplitMean::default(), validation_scored: 0, validation_refused: RefusalCounts::default(), + inventory_digest: String::new(), + cycle: None, }; assert_eq!(r.n(), 1); } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index c2eec2b..71421ff 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -624,6 +624,8 @@ fn round_over_the_dao_stores_scores_train_and_validation() { assert_eq!(r.model(), "fake-model"); assert_eq!(r.embedder(), "bag"); assert_eq!(r.chrf_version(), weksa::score::CHRF_VERSION); + assert_eq!(r.inventory_digest(), pinned.inventory_digest); + assert_eq!(r.cycle(), None, "a plain round belongs to no cycle"); // It is stored, and the packets are in the round's own store, made by the encoder step. assert_eq!( @@ -946,6 +948,34 @@ fn refusal_stages_are_counted() { assert_eq!(full.validation_scored(), 2); } +#[test] +fn non_utf8_encode_answer_is_a_parse_refusal() { + let mut dao = dao(); + let expected = prompts(&dao.run); + let answers: HashMap = good_answers() + .into_iter() + .map(|(n, a)| (format!("item:daodejing:{n}"), a)) + .collect(); + let round1 = dao.root.join("round-1"); + std::fs::create_dir_all(&round1).unwrap(); + for (key, p) in &expected { + let bytes = if key == "item:daodejing:3" { + vec![0xff, 0xfe, 0xfd] + } else { + answers[key].clone().into_bytes() + }; + std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap(); + } + let mut files = AnswerFiles::new(round1.clone(), "fake-model".into()); + let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); + // Item 3 is Validation: its refusal is a Parse refusal and the round stands. + let v = r.validation_refused(); + assert_eq!((v.parse, v.admit, v.lower), (1, 0, 0)); + assert_eq!((r.train_scored(), r.validation_scored()), (1, 1)); + assert_eq!(r.validation_mean().count, 2); + assert_eq!(dao.run.rounds().unwrap(), [r]); +} + /// An embedder that is down. struct Down; diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index 7cde9c2..d846e12 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n }\n}\n```", "type": "object", "properties": { "calls": { @@ -16,12 +16,25 @@ "corpus_digest": { "type": "string" }, + "cycle": { + "description": "`Some(k)` for the candidate round of cycle `k`.", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, "embedder": { "type": "string" }, "entity_digest": { "type": "string" }, + "inventory_digest": { + "description": "16 hex of the inventory file the round was scored under.", + "type": "string" + }, "inventory_rev": { "type": "integer", "format": "uint32", @@ -85,7 +98,8 @@ "validation_mean", "validation_refused", "train_scored", - "validation_scored" + "validation_scored", + "inventory_digest" ], "$defs": { "Call": { -- 2.47.3 From 6d17f45ca25aea7a4f362ad81ae61f25a9008c0f Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 05:02:42 +0000 Subject: [PATCH 03/19] The mint prompt becomes the editor's: Train blocks, the inventory listing and an EditFeedback line, nothing from the held-out split Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 173 +++++++++++++++++++++++++++++----- crates/weksa/src/run.rs | 7 ++ crates/weksa/tests/context.rs | 57 ++++++++--- 3 files changed, 204 insertions(+), 33 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 4dd5e30..152ff76 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -3,8 +3,10 @@ //! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only //! minting and per-item encoding (leak-owned-by-flow-and-scorer-3, //! loop-context-builder-2). Instruction text is [`MINT_INSTRUCTIONS`] and -//! [`ENCODE_INSTRUCTIONS`], consts in this module; an encode prompt also lists -//! the inventory at the run's revision. +//! [`ENCODE_INSTRUCTIONS`], consts in this module; both prompts also list the +//! inventory at the run's revision. A mint prompt is the editor's: it reads +//! [`TrainItems`], an [`Inventory`] and an [`EditFeedback`] and nothing else, so +//! no held-out text, key or per-item score can reach it. //! //! Item blocks are data from the run's pinned corpus, reached only through //! `RunStores`, the one public producer of `TrainItems` and `EncoderOffer` @@ -17,12 +19,21 @@ use std::marker::PhantomData; use crate::corpus::CorpusItem; use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind}; use crate::packet::EntityId; +use crate::run::LADDER_STEP; use crate::split::{Checkpoint, Split, SplitScheme, fnv1a64, hex16}; use crate::store::{CorpusStore, normalized}; -/// The instructions a mint prompt starts with. golf-loop-en edits the wording -/// here. -pub const MINT_INSTRUCTIONS: &str = "Mint glosses from the lines below."; +/// The instructions a mint prompt starts with: the editor's whole brief. Fixed +/// per run by `PROMPT_SOURCE_DIGEST`. +pub const MINT_INSTRUCTIONS: &str = "\ +Propose one edit to the inventory below, so that the lines below can be encoded and lowered to English more faithfully. +Answer with one JSON object and nothing else: no code fence, no commentary. +Shape: {\"add\": [{\"family\", \"gloss\", \"value_kind\", \"applies_to\"}], \"retire\": [ids]}; 1 to 8 changes in all, adds and retires together. +- family: one of Concept, Frame, Role, Relation, SpeechAct, Feature, ReferentKind, Extension. +- gloss: 1 to 120 bytes naming a general meaning, never a quoted line. +- value_kind and applies_to only on a Feature; value_kind only on an Extension. value_kind is \"Grade\", \"Flag\" or {\"Options\": [labels]}; applies_to is a list of slots. +- ids and revision numbers are the store's: write an id only to retire a live entry, by the id the inventory lists. +The line after this brief says whether your last edit was kept and the held-out composite score at the current inventory."; /// The instructions an encode prompt starts with: the encoder's whole brief. /// Fixed per run by `PROMPT_SOURCE_DIGEST`. @@ -214,6 +225,66 @@ impl<'cp> EncoderOffer<'cp> { } } +/// What the editor is told of how its last edit fared: whether it was kept, and +/// the held-out composite at the current inventory, rounded to the nearest +/// [`LADDER_STEP`]. The head composite changes only when an edit is accepted and +/// an accept needs a gain above one step, so a run releases at most +/// `1 / LADDER_STEP` distinct values (Blum and Hardt 2015, the Ladder). Nothing +/// else about the held-out split is a field. +/// +/// ``` +/// let f = weksa::context::EditFeedback::none(); +/// # let _ = f; +/// ``` +/// +/// Its fields are private: +/// +/// ```compile_fail,E0451 +/// let _ = weksa::context::EditFeedback { last: None, head: None }; +/// ``` +/// +/// and only the run builds one with a reading: +/// +/// ```compile_fail,E0624 +/// let _ = weksa::context::EditFeedback::new(Some(true), Some(0.5)); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EditFeedback { + last: Option, + head: Option, +} + +impl EditFeedback { + /// No edit yet and no reading. + pub fn none() -> Self { + Self { + last: None, + head: None, + } + } + + // Called by run::cycle (cut commit 5 of golf-loop-edit-accept). + #[allow(dead_code)] + /// `last`: whether the previous edit was accepted; `head`: the composite at + /// the current inventory, rounded here to the nearest [`LADDER_STEP`]. + pub(crate) fn new(last: Option, head: Option) -> Self { + Self { + last, + head: head.map(|x| (x / LADDER_STEP).round() * LADDER_STEP), + } + } + + fn line(&self) -> String { + let last = match self.last { + Some(true) => "accepted", + Some(false) => "rejected", + None => "none", + }; + let head = self.head.map_or("none".to_string(), |x| format!("{x:.3}")); + format!("last edit: {last}; validation composite at head: {head}") + } +} + /// Text for a model call, and the keys of the items it holds. The only /// constructors are [`Prompt::mint`] and [`Prompt::encode`]. /// @@ -224,23 +295,30 @@ impl<'cp> EncoderOffer<'cp> { /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); /// let t = run.train().unwrap(); -/// let p = weksa::context::Prompt::mint(&t); -/// assert_eq!(p.text(), weksa::context::MINT_INSTRUCTIONS); +/// let inv = run.inventory().inventory().unwrap(); +/// let feedback = weksa::context::EditFeedback::none(); +/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback); +/// assert!(p.text().starts_with(weksa::context::MINT_INSTRUCTIONS)); +/// assert!(p.text().contains("last edit: none; validation composite at head: none")); +/// assert!(p.text().contains("inventory rev:0")); /// ``` /// -/// Instructions are not a parameter: +/// Instructions are not a parameter; the same call with a fifth argument fails +/// on arity alone: /// -/// ```compile_fail +/// ```compile_fail,E0061 /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); /// let t = run.train().unwrap(); -/// let p = weksa::context::Prompt::mint("x", &t); +/// let inv = run.inventory().inventory().unwrap(); +/// let feedback = weksa::context::EditFeedback::none(); +/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback, "x"); /// ``` /// -/// ```compile_fail +/// ```compile_fail,E0277 /// fn s(_: &T) {} /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); @@ -248,7 +326,9 @@ impl<'cp> EncoderOffer<'cp> { /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); /// let t = run.train().unwrap(); -/// let p = weksa::context::Prompt::mint(&t); +/// let inv = run.inventory().inventory().unwrap(); +/// let feedback = weksa::context::EditFeedback::none(); +/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback); /// s(&p); /// ``` pub struct Prompt { @@ -303,13 +383,33 @@ fn entry_line(entry: &InventoryEntry) -> String { line } +/// Each entry of `inventory` live at `rev`, one `\n` and one line apiece: the +/// listing both prompts print under `inventory rev:`. +fn inventory_lines(inventory: &Inventory, rev: u32) -> String { + inventory + .entries() + .filter(|e| e.live_at(rev)) + .map(|e| format!("\n{}", entry_line(e))) + .collect() +} + /// Printed in place of a prior turn whose text no live Train item holds. const ELIDED: &str = "[elided]"; impl Prompt { - /// [`MINT_INSTRUCTIONS`], then every Train item in key order. - pub fn mint(train: &TrainItems) -> Prompt { - let mut parts = vec![MINT_INSTRUCTIONS.to_string()]; + /// [`MINT_INSTRUCTIONS`], the feedback line, the entries of `inventory` live + /// at `rev`, then every Train item in key order without prior turns. + pub fn mint( + train: &TrainItems, + inventory: &Inventory, + rev: u32, + feedback: &EditFeedback, + ) -> Prompt { + let mut parts = vec![ + MINT_INSTRUCTIONS.to_string(), + feedback.line(), + format!("inventory rev:{rev}{}", inventory_lines(inventory, rev)), + ]; parts.extend(train.items.iter().map(|item| block(item, None))); Prompt { text: parts.join("\n\n"), @@ -327,15 +427,12 @@ impl Prompt { rev: u32, ) -> Prompt { let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect(); - let mut text = format!( - "{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}\n\ninventory rev:{rev}", + let text = format!( + "{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}\n\ninventory rev:{rev}{}", block(&offer.item, Some(train)), - ids.join(", ") + ids.join(", "), + inventory_lines(inventory, rev) ); - for entry in inventory.entries().filter(|e| e.live_at(rev)) { - text.push('\n'); - text.push_str(&entry_line(entry)); - } Prompt { text, items: vec![offer.item.id.clone()], @@ -363,6 +460,38 @@ mod tests { use super::*; use crate::split::fixture::{group_in, item}; + #[test] + fn the_feedback_line_prints_exact_values() { + let line = |last, head| EditFeedback::new(last, head).line(); + assert_eq!( + EditFeedback::none().line(), + "last edit: none; validation composite at head: none" + ); + assert_eq!( + line(Some(true), Some(0.5)), + "last edit: accepted; validation composite at head: 0.500" + ); + assert_eq!( + line(Some(false), Some(0.0)), + "last edit: rejected; validation composite at head: 0.000" + ); + assert_eq!(line(None, None), EditFeedback::none().line()); + // The head is rounded to the nearest step before it is printed. + assert_eq!( + line(None, Some(0.5012)), + "last edit: none; validation composite at head: 0.500" + ); + assert_eq!( + line(None, Some(0.5026)), + "last edit: none; validation composite at head: 0.505" + ); + assert_eq!( + line(None, Some(0.9999)), + "last edit: none; validation composite at head: 1.000" + ); + assert_eq!(EditFeedback::new(None, Some(0.5026)).head, Some(0.505)); + } + #[test] fn checkpoint_offer_gate() { // A fresh checkpoint per call: this test is the gate, not the once rule. diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index a8ea6d6..4120100 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -41,6 +41,13 @@ pub const PROMPT_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("context.rs")); /// included, ends a run; code outside that file is not covered. pub const LOWERER_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("lower.rs")); +/// The least composite gain an edit must show over item noise to be kept, and +/// the grid the editor's one held-out number is rounded to. A run releases at +/// most `1 / LADDER_STEP` = 200 distinct head composites (the Ladder, Blum and +/// Hardt 2015), because the head changes only on an accept and an accept needs +/// a gain above one step. +pub const LADDER_STEP: f64 = 0.005; + /// What a run pinned at open. Key `run: