diff --git a/README.md b/README.md index d327e48..e9aa0e9 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,9 @@ measured. - `settings/`: setting-specific proving grounds for language projects. **Live versus planned.** Live: docs, fixtures, examples, ontology data and the -packet crate `crates/weksa`, including the scoring library. Planned: lowering, -the `weksa score` subcommand, and the golf loop that scores lowered English against an expressive corpus. No +packet crate `crates/weksa`, including lowering, `weksa lower` and `weksa score`, +and the golf loop's round and edit-accept cycle (`weksa loop open|round|cycle`). +Planned: a model lowerer, Score documents and the other panel languages. No daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`. ## Ownership Model diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 4dd5e30..eb1f6bc 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -3,8 +3,10 @@ //! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only //! minting and per-item encoding (leak-owned-by-flow-and-scorer-3, //! loop-context-builder-2). Instruction text is [`MINT_INSTRUCTIONS`] and -//! [`ENCODE_INSTRUCTIONS`], consts in this module; an encode prompt also lists -//! the inventory at the run's revision. +//! [`ENCODE_INSTRUCTIONS`], consts in this module; both prompts also list the +//! inventory at the run's revision. A mint prompt is the editor's: it reads +//! [`TrainItems`], an [`Inventory`] and an [`EditFeedback`] and nothing else, so +//! no held-out text, key or per-item score can reach it. //! //! Item blocks are data from the run's pinned corpus, reached only through //! `RunStores`, the one public producer of `TrainItems` and `EncoderOffer` @@ -17,12 +19,21 @@ use std::marker::PhantomData; use crate::corpus::CorpusItem; use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind}; use crate::packet::EntityId; +use crate::run::LADDER_STEP; use crate::split::{Checkpoint, Split, SplitScheme, fnv1a64, hex16}; use crate::store::{CorpusStore, normalized}; -/// The instructions a mint prompt starts with. golf-loop-en edits the wording -/// here. -pub const MINT_INSTRUCTIONS: &str = "Mint glosses from the lines below."; +/// The instructions a mint prompt starts with: the editor's whole brief. Fixed +/// per run by `PROMPT_SOURCE_DIGEST`. +pub const MINT_INSTRUCTIONS: &str = "\ +Propose one edit to the inventory below, so that the lines below can be encoded and lowered to English more faithfully. +Answer with one JSON object and nothing else: no code fence, no commentary. +Shape: {\"add\": [{\"family\", \"gloss\", \"value_kind\", \"applies_to\"}], \"retire\": [ids]}; 1 to 8 changes in all, adds and retires together. +- family: one of Concept, Frame, Role, Relation, SpeechAct, Feature, ReferentKind, Extension. +- gloss: 1 to 120 bytes naming a general meaning, never a quoted line. +- value_kind and applies_to only on a Feature; value_kind only on an Extension. value_kind is \"Grade\", \"Flag\" or {\"Options\": [labels]}; applies_to is a list of slots. +- ids and revision numbers are the store's: write an id only to retire a live entry, by the id the inventory lists. +The line after this brief says whether your last edit was kept and the held-out composite score at the current inventory."; /// The instructions an encode prompt starts with: the encoder's whole brief. /// Fixed per run by `PROMPT_SOURCE_DIGEST`. @@ -214,6 +225,64 @@ impl<'cp> EncoderOffer<'cp> { } } +/// What the editor is told of how its last edit fared: whether it was kept, and +/// the held-out composite at the current inventory, rounded to the nearest +/// [`LADDER_STEP`]. The head composite changes only when an edit is accepted and +/// an accept needs a gain above one step, so a run releases at most +/// `1 / LADDER_STEP` distinct values (Blum and Hardt 2015, the Ladder). Nothing +/// else about the held-out split is a field. +/// +/// ``` +/// let f = weksa::context::EditFeedback::none(); +/// # let _ = f; +/// ``` +/// +/// Its fields are private: +/// +/// ```compile_fail,E0451 +/// let _ = weksa::context::EditFeedback { last: None, head: None }; +/// ``` +/// +/// and only the run builds one with a reading: +/// +/// ```compile_fail,E0624 +/// let _ = weksa::context::EditFeedback::new(Some(true), Some(0.5)); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EditFeedback { + last: Option, + head: Option, +} + +impl EditFeedback { + /// No edit yet and no reading. + pub fn none() -> Self { + Self { + last: None, + head: None, + } + } + + /// `last`: whether the previous edit was accepted; `head`: the composite at + /// the current inventory, rounded here to the nearest [`LADDER_STEP`]. + pub(crate) fn new(last: Option, head: Option) -> Self { + Self { + last, + head: head.map(|x| (x / LADDER_STEP).round() * LADDER_STEP), + } + } + + fn line(&self) -> String { + let last = match self.last { + Some(true) => "accepted", + Some(false) => "rejected", + None => "none", + }; + let head = self.head.map_or("none".to_string(), |x| format!("{x:.3}")); + format!("last edit: {last}; validation composite at head: {head}") + } +} + /// Text for a model call, and the keys of the items it holds. The only /// constructors are [`Prompt::mint`] and [`Prompt::encode`]. /// @@ -224,23 +293,30 @@ impl<'cp> EncoderOffer<'cp> { /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); /// let t = run.train().unwrap(); -/// let p = weksa::context::Prompt::mint(&t); -/// assert_eq!(p.text(), weksa::context::MINT_INSTRUCTIONS); +/// let inv = run.inventory().inventory().unwrap(); +/// let feedback = weksa::context::EditFeedback::none(); +/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback); +/// assert!(p.text().starts_with(weksa::context::MINT_INSTRUCTIONS)); +/// assert!(p.text().contains("last edit: none; validation composite at head: none")); +/// assert!(p.text().contains("inventory rev:0")); /// ``` /// -/// Instructions are not a parameter: +/// Instructions are not a parameter; the same call with a fifth argument fails +/// on arity alone: /// -/// ```compile_fail +/// ```compile_fail,E0061 /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); /// let t = run.train().unwrap(); -/// let p = weksa::context::Prompt::mint("x", &t); +/// let inv = run.inventory().inventory().unwrap(); +/// let feedback = weksa::context::EditFeedback::none(); +/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback, "x"); /// ``` /// -/// ```compile_fail +/// ```compile_fail,E0277 /// fn s(_: &T) {} /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); @@ -248,7 +324,9 @@ impl<'cp> EncoderOffer<'cp> { /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; /// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap(); /// let t = run.train().unwrap(); -/// let p = weksa::context::Prompt::mint(&t); +/// let inv = run.inventory().inventory().unwrap(); +/// let feedback = weksa::context::EditFeedback::none(); +/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback); /// s(&p); /// ``` pub struct Prompt { @@ -303,13 +381,33 @@ fn entry_line(entry: &InventoryEntry) -> String { line } +/// Each entry of `inventory` live at `rev`, one `\n` and one line apiece: the +/// listing both prompts print under `inventory rev:`. +fn inventory_lines(inventory: &Inventory, rev: u32) -> String { + inventory + .entries() + .filter(|e| e.live_at(rev)) + .map(|e| format!("\n{}", entry_line(e))) + .collect() +} + /// Printed in place of a prior turn whose text no live Train item holds. const ELIDED: &str = "[elided]"; impl Prompt { - /// [`MINT_INSTRUCTIONS`], then every Train item in key order. - pub fn mint(train: &TrainItems) -> Prompt { - let mut parts = vec![MINT_INSTRUCTIONS.to_string()]; + /// [`MINT_INSTRUCTIONS`], the feedback line, the entries of `inventory` live + /// at `rev`, then every Train item in key order without prior turns. + pub fn mint( + train: &TrainItems, + inventory: &Inventory, + rev: u32, + feedback: &EditFeedback, + ) -> Prompt { + let mut parts = vec![ + MINT_INSTRUCTIONS.to_string(), + feedback.line(), + format!("inventory rev:{rev}{}", inventory_lines(inventory, rev)), + ]; parts.extend(train.items.iter().map(|item| block(item, None))); Prompt { text: parts.join("\n\n"), @@ -327,15 +425,12 @@ impl Prompt { rev: u32, ) -> Prompt { let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect(); - let mut text = format!( - "{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}\n\ninventory rev:{rev}", + let text = format!( + "{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}\n\ninventory rev:{rev}{}", block(&offer.item, Some(train)), - ids.join(", ") + ids.join(", "), + inventory_lines(inventory, rev) ); - for entry in inventory.entries().filter(|e| e.live_at(rev)) { - text.push('\n'); - text.push_str(&entry_line(entry)); - } Prompt { text, items: vec![offer.item.id.clone()], @@ -363,6 +458,38 @@ mod tests { use super::*; use crate::split::fixture::{group_in, item}; + #[test] + fn the_feedback_line_prints_exact_values() { + let line = |last, head| EditFeedback::new(last, head).line(); + assert_eq!( + EditFeedback::none().line(), + "last edit: none; validation composite at head: none" + ); + assert_eq!( + line(Some(true), Some(0.5)), + "last edit: accepted; validation composite at head: 0.500" + ); + assert_eq!( + line(Some(false), Some(0.0)), + "last edit: rejected; validation composite at head: 0.000" + ); + assert_eq!(line(None, None), EditFeedback::none().line()); + // The head is rounded to the nearest step before it is printed. + assert_eq!( + line(None, Some(0.5012)), + "last edit: none; validation composite at head: 0.500" + ); + assert_eq!( + line(None, Some(0.5026)), + "last edit: none; validation composite at head: 0.505" + ); + assert_eq!( + line(None, Some(0.9999)), + "last edit: none; validation composite at head: 1.000" + ); + assert_eq!(EditFeedback::new(None, Some(0.5026)).head, Some(0.505)); + } + #[test] fn checkpoint_offer_gate() { // A fresh checkpoint per call: this test is the gate, not the once rule. diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 430f360..c8b37a2 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -1,15 +1,23 @@ -//! The encoder step: what a model's answer becomes. This is the one place JSON -//! is read (the xenos boundary): a model answers in JSON, `PacketDraft::parse` -//! reads it, and nothing is stored as JSON. The model writes meaning only; -//! `into_packet` sets the id, schema version, inventory revision, kind, +//! The encoder and editor steps: what a model's answer becomes. This is the one +//! place JSON is read (the xenos boundary): a model answers in JSON, +//! `PacketDraft::parse` or `EditDraft::parse` reads it (both through +//! `from_json`), and nothing is stored as JSON. The encoder writes meaning +//! only; `into_packet` sets the id, schema version, inventory revision, kind, //! provenance, constraints and extensions, and an empty trace, so a model that -//! saw a Validation item cannot hand its words to whoever reads packets. +//! saw a Validation item cannot hand its words to whoever reads packets. The +//! editor writes family, gloss, value kind and slots only; `into_revision` sets +//! every id and revision number. use std::fmt; +use std::num::NonZeroU32; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use crate::corpus::CorpusItemId; +use crate::inventory::{ + Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind, +}; use crate::packet::{ Constraints, Context, Discourse, EncoderId, Grade, Packet, PacketId, PacketKind, Predication, Provenance, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, Trace, @@ -58,12 +66,17 @@ impl fmt::Debug for DraftError { impl std::error::Error for DraftError {} +/// The one JSON read of a model's answer. +fn from_json(answer: &str) -> Result { + serde_json::from_str(answer.trim()).map_err(|e| DraftError { + line: e.line(), + column: e.column(), + }) +} + impl PacketDraft { pub fn parse(answer: &str) -> Result { - serde_json::from_str(answer.trim()).map_err(|e| DraftError { - line: e.line(), - column: e.column(), - }) + from_json(answer) } /// The draft that says what `packet` says. @@ -105,6 +118,87 @@ impl PacketDraft { } } +/// The most changes (adds and retires together) one edit may make. +pub const MAX_EDIT_CHANGES: usize = 8; + +/// One entry an edit adds, as the editor writes it: no id and no revision. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EntryDraft { + pub family: Family, + pub gloss: String, + #[serde(default)] + pub value_kind: Option, + #[serde(default)] + pub applies_to: Vec, +} + +/// An inventory edit as the editor writes it: entries to add and ids to retire. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EditDraft { + #[serde(default)] + pub add: Vec, + #[serde(default)] + pub retire: Vec, +} + +impl EditDraft { + pub fn parse(answer: &str) -> Result { + from_json(answer) + } + + /// The revision after `base` this edit asks for, and the entries it adds. + /// `None` when the edit makes no change or more than [`MAX_EDIT_CHANGES`]. + /// Every id is the store's: an added entry takes the next number of its + /// family after the highest in `inventory`, retired entries included, in + /// draft order. Nothing else is judged here: the gloss, the entry's shape, + /// whether a retire target is live and a repeated id are + /// `InventoryStore::admit_revision`'s to refuse. + pub(crate) fn into_revision( + self, + inventory: &Inventory, + base: u32, + ) -> Option<(InventoryRevision, Vec)> { + let count = self.add.len() + self.retire.len(); + if count == 0 || count > MAX_EDIT_CHANGES { + return None; + } + let n = base.checked_add(1)?; + let mut entries: Vec = Vec::new(); + for draft in self.add { + let highest = inventory + .entries() + .map(|e| e.id) + .chain(entries.iter().map(|e| e.id)) + .filter(|id| id.family() == draft.family) + .map(|id| id.n().get()) + .max() + .unwrap_or(0); + entries.push(InventoryEntry { + id: draft.family.id(NonZeroU32::new(highest.checked_add(1)?)?), + gloss: draft.gloss, + value_kind: draft.value_kind, + applies_to: draft.applies_to, + introduced_rev: n, + retired_rev: None, + }); + } + let changes = entries + .iter() + .map(|e| Change::Add(e.id)) + .chain(self.retire.into_iter().map(Change::Retire)) + .collect(); + let revision = InventoryRevision { + n, + parent: Some(base), + changes, + frozen: false, + }; + Some((revision, entries)) + } +} + /// `encoder:`, a digest of the model id and the prompt digest. pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId { let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes()); @@ -228,4 +322,147 @@ mod tests { assert!(!text.contains("CANARY"), "{text}"); } } + + fn entry(id: &str, retired: Option) -> InventoryEntry { + let id = InvId::parse(id).unwrap(); + InventoryEntry { + id, + gloss: "g".into(), + value_kind: (id.family() == Family::Extension).then_some(ValueKind::Flag), + applies_to: vec![], + introduced_rev: 1, + retired_rev: retired, + } + } + + fn add(family: Family, gloss: &str) -> EntryDraft { + EntryDraft { + family, + gloss: gloss.into(), + value_kind: None, + applies_to: vec![], + } + } + + #[test] + fn ids_are_the_stores() { + // concept:1 live, concept:3 retired (the highest), frame:2 live; no role. + let inv = Inventory::new( + vec![ + entry("concept:1", None), + entry("concept:3", Some(2)), + entry("frame:2", None), + ], + vec![], + ); + let draft = EditDraft { + add: vec![ + add(Family::Concept, "first"), + add(Family::Role, "role"), + add(Family::Concept, "second"), + add(Family::Frame, "frame"), + ], + retire: vec![ + InvId::parse("concept:1").unwrap(), + InvId::parse("frame:2").unwrap(), + ], + }; + let (rev, entries) = draft.clone().into_revision(&inv, 6).unwrap(); + let ids: Vec = entries.iter().map(|e| e.id.to_string()).collect(); + // After the highest of the family, retired ones included, in draft order. + assert_eq!(ids, ["concept:4", "role:1", "concept:5", "frame:3"]); + assert_eq!((rev.n, rev.parent, rev.frozen), (7, Some(6), false)); + assert!( + entries + .iter() + .all(|e| e.introduced_rev == 7 && e.retired_rev.is_none()) + ); + let glosses: Vec<&str> = entries.iter().map(|e| e.gloss.as_str()).collect(); + assert_eq!(glosses, ["first", "role", "second", "frame"]); + // Adds in draft order, then the retires in draft order. + let changes: Vec = rev + .changes + .iter() + .map(|c| match c { + Change::Add(id) => format!("+{id}"), + Change::Retire(id) => format!("-{id}"), + }) + .collect(); + assert_eq!( + changes, + [ + "+concept:4", + "+role:1", + "+concept:5", + "+frame:3", + "-concept:1", + "-frame:2" + ] + ); + } + + #[test] + fn the_change_count_is_one_to_eight() { + let inv = Inventory::new(vec![entry("concept:1", None)], vec![]); + let adds = |n: usize| EditDraft { + add: (0..n) + .map(|i| add(Family::Concept, &format!("g{i}"))) + .collect(), + retire: vec![], + }; + assert!(adds(0).into_revision(&inv, 1).is_none()); + assert!(adds(1).into_revision(&inv, 1).is_some()); + assert_eq!(adds(8).into_revision(&inv, 1).unwrap().1.len(), 8); + assert!(adds(9).into_revision(&inv, 1).is_none()); + // Adds and retires count together. + let mixed = |a: usize, r: usize| EditDraft { + add: (0..a).map(|_| add(Family::Concept, "g")).collect(), + retire: vec![InvId::parse("concept:1").unwrap(); r], + }; + assert!(mixed(4, 4).into_revision(&inv, 1).is_some()); + assert!(mixed(4, 5).into_revision(&inv, 1).is_none()); + assert!(mixed(0, 9).into_revision(&inv, 1).is_none()); + // A repeated retire is the door's to refuse, not this fn's. + assert!(mixed(0, 2).into_revision(&inv, 1).is_some()); + } + + #[test] + fn an_edit_draft_reads_what_an_editor_may_write() { + let d = EditDraft::parse( + r#" {"add":[{"family":"Feature","gloss":"g","value_kind":{"Options":["a","b"]},"applies_to":["Referent"]}],"retire":["concept:2"]} "#, + ) + .unwrap(); + assert_eq!(d.add[0].family, Family::Feature); + assert_eq!(d.add[0].applies_to, [Slot::Referent]); + assert_eq!(d.retire, [InvId::parse("concept:2").unwrap()]); + // Both lists default; value_kind and applies_to default. + let d = EditDraft::parse(r#"{"add":[{"family":"Concept","gloss":"g"}]}"#).unwrap(); + assert_eq!(d.add[0].value_kind, None); + assert!(d.add[0].applies_to.is_empty() && d.retire.is_empty()); + assert_eq!(EditDraft::parse("{}").unwrap().add.len(), 0); + } + + #[test] + fn an_edit_draft_refuses_what_the_store_writes() { + // An id, a revision number or a retired mark is the store's to write. + for field in ["id", "introduced_rev", "retired_rev", "n", "parent"] { + let json = format!(r#"{{"add":[{{"family":"Concept","gloss":"g","{field}":1}}]}}"#); + assert!(EditDraft::parse(&json).is_err(), "{field} in an entry"); + } + for field in ["revision", "frozen", "changes"] { + let json = format!(r#"{{"add":[],"{field}":1}}"#); + assert!(EditDraft::parse(&json).is_err(), "{field} in an edit"); + } + // A family or a retire id that is not one. + assert!(EditDraft::parse(r#"{"add":[{"family":"Nope","gloss":"g"}]}"#).is_err()); + assert!(EditDraft::parse(r#"{"retire":["concept:0"]}"#).is_err()); + assert!(EditDraft::parse(r#"{"retire":[1]}"#).is_err()); + // Not JSON, not an object, an error that names no token. + let err = EditDraft::parse("{\n \"add\": CANARYTOKEN").unwrap_err(); + assert_eq!((err.line, err.column), (2, 10)); + for text in [err.to_string(), format!("{err:?}")] { + assert!(!text.contains("CANARY"), "{text}"); + } + assert!(EditDraft::parse("3").is_err() && EditDraft::parse("\"x\"").is_err()); + } } diff --git a/crates/weksa/src/inventory.rs b/crates/weksa/src/inventory.rs index 6c8b984..ec59b13 100644 --- a/crates/weksa/src/inventory.rs +++ b/crates/weksa/src/inventory.rs @@ -85,6 +85,11 @@ impl InvId { pub fn family(self) -> Family { self.family } + + /// The `n` of `:`. + pub fn n(self) -> NonZeroU32 { + self.n + } } impl fmt::Display for InvId { diff --git a/crates/weksa/src/lib.rs b/crates/weksa/src/lib.rs index f61a9b6..c741c62 100644 --- a/crates/weksa/src/lib.rs +++ b/crates/weksa/src/lib.rs @@ -23,7 +23,7 @@ pub mod store; pub mod validate; pub use corpus::{CorpusItem, CorpusItemId, NameSpan, Source, Turn}; -pub use encode::PacketDraft; +pub use encode::{EditDraft, PacketDraft}; pub use entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; pub use form::FormError; pub use inventory::{ @@ -36,7 +36,7 @@ pub use packet::{ EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance, }; pub use phones::{Phone, PhoneInventory, SegmentError, Stress}; -pub use run::{RoundRecord, RunStores, round}; +pub use run::{Decision, RoundRecord, RunStores, cycle, round}; pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf}; pub use speaker::{Annotator, PairRelation, Party, Scene, Speaker}; pub use split::{Split, SplitScheme}; diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 5e7eb28..7901ca5 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{RoundOutcome, RunPaths, RunStores, round}; +use weksa::run::{CycleOutcome, RoundOutcome, RunPaths, RunStores, cycle, round}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, @@ -98,6 +98,15 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Run one edit-accept cycle of the run at ROOT. The stores are read only + /// from the run's record. Prompts without answers are written to the + /// cycle's directory and the command exits 3; rerun after the `.answer` + /// files exist. + Cycle { + root: PathBuf, + #[arg(long, default_value = "http://127.0.0.1:11434")] + ollama_url: String, + }, } #[derive(Subcommand)] @@ -758,6 +767,25 @@ fn main() -> Result<()> { RoundOutcome::Recorded(r) => println!("{}", r.summary_line()), } } + Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { + let unreadable = |_| anyhow::anyhow!("a store would not read"); + let mut run = RunStores::open(&root)?; + let mut files = AnswerFiles::new( + run.next_cycle_dir().map_err(unreadable)?, + run.record().model.clone(), + ); + let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder)); + match cycle(&mut run, &mut files, &scorer)? { + CycleOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + CycleOutcome::Decided(decision) => println!("{}", decision.summary_line()), + } + } Command::Schema { out } => { std::fs::create_dir_all(&out)?; for (name, json) in weksa::schema::published() { diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index de6d846..1beb442 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -1,8 +1,15 @@ -//! One golf-loop run: the stores it pinned at open, the rounds it has recorded, -//! and the one writer of both (`RunStores::create` and, for rounds, `round`). +//! One golf-loop run: the stores it pinned at open, the rounds and decisions it +//! has recorded, and the one writer of all of them (`RunStores::create`, `round` +//! and `cycle`). A cycle asks an editor for one inventory edit, scores a +//! candidate round under it and accepts or rejects on Validation (`decide`); the +//! latest Accepted decision names the run's head inventory. //! //! A run pins its corpus and entity store files by canonical path and FNV-1a 64 -//! digest of their bytes. Every round reopens them only from that record and +//! digest of their bytes. It works on its own copy of the inventory: +//! `RunStores::create` copies the caller's inventory file to +//! `/inventory.cc` and pins the digest of the copy; the caller's path is +//! kept as provenance and never opened again, and `open` refuses a head file +//! whose digest differs. Every round reopens them only from that record and //! refuses on drift, and every round record carries the digests. Nothing inside //! a process whose writer controls the code can stop an item faked into a //! scratch store; what this makes true is that a run over a scratch store is a @@ -19,9 +26,9 @@ use cultcache_rs::{CultCache, DatabaseEntry}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::context::{EncoderOffer, Prompt, TrainItems}; +use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; use crate::corpus::{CorpusItem, CorpusItemId}; -use crate::encode::{PacketDraft, encoder_id}; +use crate::encode::{EditDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::lower::lower; use crate::model::{ModelPort, PortError}; @@ -37,6 +44,13 @@ pub const PROMPT_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("context.rs")); /// included, ends a run; code outside that file is not covered. pub const LOWERER_SOURCE_DIGEST: u64 = fnv1a64(include_bytes!("lower.rs")); +/// The least composite gain an edit must show over item noise to be kept, and +/// the grid the editor's one held-out number is rounded to. A run releases at +/// most `1 / LADDER_STEP` = 200 distinct head composites (the Ladder, Blum and +/// Hardt 2015), because the head changes only on an accept and an accept needs +/// a gain above one step. +pub const LADDER_STEP: f64 = 0.005; + /// What a run pinned at open. Key `run: