golf-loop-edit-accept: a Train-only editor proposes an inventory edit, a candidate round scores it, a noise-floor rule accepts or rejects on Validation #30
16 changed files with 2900 additions and 94 deletions
|
|
@ -32,8 +32,9 @@ measured.
|
|||
- `settings/`: setting-specific proving grounds for language projects.
|
||||
|
||||
**Live versus planned.** Live: docs, fixtures, examples, ontology data and the
|
||||
packet crate `crates/weksa`, including the scoring library. Planned: lowering,
|
||||
the `weksa score` subcommand, and the golf loop that scores lowered English against an expressive corpus. No
|
||||
packet crate `crates/weksa`, including lowering, `weksa lower` and `weksa score`,
|
||||
and the golf loop's round and edit-accept cycle (`weksa loop open|round|cycle`).
|
||||
Planned: a model lowerer, Score documents and the other panel languages. No
|
||||
daemon runs; the earlier Node daemon is parked at tag `parked/weksa-node-body`.
|
||||
|
||||
## Ownership Model
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@
|
|||
//! [`TrainItems`] or by encoding one item an [`EncoderOffer`] holds: Train-only
|
||||
//! minting and per-item encoding (leak-owned-by-flow-and-scorer-3,
|
||||
//! loop-context-builder-2). Instruction text is [`MINT_INSTRUCTIONS`] and
|
||||
//! [`ENCODE_INSTRUCTIONS`], consts in this module; an encode prompt also lists
|
||||
//! the inventory at the run's revision.
|
||||
//! [`ENCODE_INSTRUCTIONS`], consts in this module; both prompts also list the
|
||||
//! inventory at the run's revision. A mint prompt is the editor's: it reads
|
||||
//! [`TrainItems`], an [`Inventory`] and an [`EditFeedback`] and nothing else, so
|
||||
//! no held-out text, key or per-item score can reach it.
|
||||
//!
|
||||
//! Item blocks are data from the run's pinned corpus, reached only through
|
||||
//! `RunStores`, the one public producer of `TrainItems` and `EncoderOffer`
|
||||
|
|
@ -17,12 +19,21 @@ use std::marker::PhantomData;
|
|||
use crate::corpus::CorpusItem;
|
||||
use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind};
|
||||
use crate::packet::EntityId;
|
||||
use crate::run::LADDER_STEP;
|
||||
use crate::split::{Checkpoint, Split, SplitScheme, fnv1a64, hex16};
|
||||
use crate::store::{CorpusStore, normalized};
|
||||
|
||||
/// The instructions a mint prompt starts with. golf-loop-en edits the wording
|
||||
/// here.
|
||||
pub const MINT_INSTRUCTIONS: &str = "Mint glosses from the lines below.";
|
||||
/// The instructions a mint prompt starts with: the editor's whole brief. Fixed
|
||||
/// per run by `PROMPT_SOURCE_DIGEST`.
|
||||
pub const MINT_INSTRUCTIONS: &str = "\
|
||||
Propose one edit to the inventory below, so that the lines below can be encoded and lowered to English more faithfully.
|
||||
Answer with one JSON object and nothing else: no code fence, no commentary.
|
||||
Shape: {\"add\": [{\"family\", \"gloss\", \"value_kind\", \"applies_to\"}], \"retire\": [ids]}; 1 to 8 changes in all, adds and retires together.
|
||||
- family: one of Concept, Frame, Role, Relation, SpeechAct, Feature, ReferentKind, Extension.
|
||||
- gloss: 1 to 120 bytes naming a general meaning, never a quoted line.
|
||||
- value_kind and applies_to only on a Feature; value_kind only on an Extension. value_kind is \"Grade\", \"Flag\" or {\"Options\": [labels]}; applies_to is a list of slots.
|
||||
- ids and revision numbers are the store's: write an id only to retire a live entry, by the id the inventory lists.
|
||||
The line after this brief says whether your last edit was kept and the held-out composite score at the current inventory.";
|
||||
|
||||
/// The instructions an encode prompt starts with: the encoder's whole brief.
|
||||
/// Fixed per run by `PROMPT_SOURCE_DIGEST`.
|
||||
|
|
@ -214,6 +225,64 @@ impl<'cp> EncoderOffer<'cp> {
|
|||
}
|
||||
}
|
||||
|
||||
/// What the editor is told of how its last edit fared: whether it was kept, and
|
||||
/// the held-out composite at the current inventory, rounded to the nearest
|
||||
/// [`LADDER_STEP`]. The head composite changes only when an edit is accepted and
|
||||
/// an accept needs a gain above one step, so a run releases at most
|
||||
/// `1 / LADDER_STEP` distinct values (Blum and Hardt 2015, the Ladder). Nothing
|
||||
/// else about the held-out split is a field.
|
||||
///
|
||||
/// ```
|
||||
/// let f = weksa::context::EditFeedback::none();
|
||||
/// # let _ = f;
|
||||
/// ```
|
||||
///
|
||||
/// Its fields are private:
|
||||
///
|
||||
/// ```compile_fail,E0451
|
||||
/// let _ = weksa::context::EditFeedback { last: None, head: None };
|
||||
/// ```
|
||||
///
|
||||
/// and only the run builds one with a reading:
|
||||
///
|
||||
/// ```compile_fail,E0624
|
||||
/// let _ = weksa::context::EditFeedback::new(Some(true), Some(0.5));
|
||||
/// ```
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct EditFeedback {
|
||||
last: Option<bool>,
|
||||
head: Option<f64>,
|
||||
}
|
||||
|
||||
impl EditFeedback {
|
||||
/// No edit yet and no reading.
|
||||
pub fn none() -> Self {
|
||||
Self {
|
||||
last: None,
|
||||
head: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `last`: whether the previous edit was accepted; `head`: the composite at
|
||||
/// the current inventory, rounded here to the nearest [`LADDER_STEP`].
|
||||
pub(crate) fn new(last: Option<bool>, head: Option<f64>) -> Self {
|
||||
Self {
|
||||
last,
|
||||
head: head.map(|x| (x / LADDER_STEP).round() * LADDER_STEP),
|
||||
}
|
||||
}
|
||||
|
||||
fn line(&self) -> String {
|
||||
let last = match self.last {
|
||||
Some(true) => "accepted",
|
||||
Some(false) => "rejected",
|
||||
None => "none",
|
||||
};
|
||||
let head = self.head.map_or("none".to_string(), |x| format!("{x:.3}"));
|
||||
format!("last edit: {last}; validation composite at head: {head}")
|
||||
}
|
||||
}
|
||||
|
||||
/// Text for a model call, and the keys of the items it holds. The only
|
||||
/// constructors are [`Prompt::mint`] and [`Prompt::encode`].
|
||||
///
|
||||
|
|
@ -224,23 +293,30 @@ impl<'cp> EncoderOffer<'cp> {
|
|||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let p = weksa::context::Prompt::mint(&t);
|
||||
/// assert_eq!(p.text(), weksa::context::MINT_INSTRUCTIONS);
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback);
|
||||
/// assert!(p.text().starts_with(weksa::context::MINT_INSTRUCTIONS));
|
||||
/// assert!(p.text().contains("last edit: none; validation composite at head: none"));
|
||||
/// assert!(p.text().contains("inventory rev:0"));
|
||||
/// ```
|
||||
///
|
||||
/// Instructions are not a parameter:
|
||||
/// Instructions are not a parameter; the same call with a fifth argument fails
|
||||
/// on arity alone:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// ```compile_fail,E0061
|
||||
/// # let dir = tempfile::tempdir().unwrap();
|
||||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let p = weksa::context::Prompt::mint("x", &t);
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback, "x");
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// ```compile_fail,E0277
|
||||
/// fn s<T: serde::Serialize>(_: &T) {}
|
||||
/// # let dir = tempfile::tempdir().unwrap();
|
||||
/// # let p = |n: &str| dir.path().join(n);
|
||||
|
|
@ -248,7 +324,9 @@ impl<'cp> EncoderOffer<'cp> {
|
|||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e").unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let p = weksa::context::Prompt::mint(&t);
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
/// let p = weksa::context::Prompt::mint(&t, &inv, 0, &feedback);
|
||||
/// s(&p);
|
||||
/// ```
|
||||
pub struct Prompt {
|
||||
|
|
@ -303,13 +381,33 @@ fn entry_line(entry: &InventoryEntry) -> String {
|
|||
line
|
||||
}
|
||||
|
||||
/// Each entry of `inventory` live at `rev`, one `\n` and one line apiece: the
|
||||
/// listing both prompts print under `inventory rev:<rev>`.
|
||||
fn inventory_lines(inventory: &Inventory, rev: u32) -> String {
|
||||
inventory
|
||||
.entries()
|
||||
.filter(|e| e.live_at(rev))
|
||||
.map(|e| format!("\n{}", entry_line(e)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Printed in place of a prior turn whose text no live Train item holds.
|
||||
const ELIDED: &str = "[elided]";
|
||||
|
||||
impl Prompt {
|
||||
/// [`MINT_INSTRUCTIONS`], then every Train item in key order.
|
||||
pub fn mint(train: &TrainItems) -> Prompt {
|
||||
let mut parts = vec![MINT_INSTRUCTIONS.to_string()];
|
||||
/// [`MINT_INSTRUCTIONS`], the feedback line, the entries of `inventory` live
|
||||
/// at `rev`, then every Train item in key order without prior turns.
|
||||
pub fn mint(
|
||||
train: &TrainItems,
|
||||
inventory: &Inventory,
|
||||
rev: u32,
|
||||
feedback: &EditFeedback,
|
||||
) -> Prompt {
|
||||
let mut parts = vec![
|
||||
MINT_INSTRUCTIONS.to_string(),
|
||||
feedback.line(),
|
||||
format!("inventory rev:{rev}{}", inventory_lines(inventory, rev)),
|
||||
];
|
||||
parts.extend(train.items.iter().map(|item| block(item, None)));
|
||||
Prompt {
|
||||
text: parts.join("\n\n"),
|
||||
|
|
@ -327,15 +425,12 @@ impl Prompt {
|
|||
rev: u32,
|
||||
) -> Prompt {
|
||||
let ids: Vec<&str> = offer.entities.iter().map(EntityId::as_str).collect();
|
||||
let mut text = format!(
|
||||
"{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}\n\ninventory rev:{rev}",
|
||||
let text = format!(
|
||||
"{ENCODE_INSTRUCTIONS}\n\n{}\n\nentities: {}\n\ninventory rev:{rev}{}",
|
||||
block(&offer.item, Some(train)),
|
||||
ids.join(", ")
|
||||
ids.join(", "),
|
||||
inventory_lines(inventory, rev)
|
||||
);
|
||||
for entry in inventory.entries().filter(|e| e.live_at(rev)) {
|
||||
text.push('\n');
|
||||
text.push_str(&entry_line(entry));
|
||||
}
|
||||
Prompt {
|
||||
text,
|
||||
items: vec![offer.item.id.clone()],
|
||||
|
|
@ -363,6 +458,38 @@ mod tests {
|
|||
use super::*;
|
||||
use crate::split::fixture::{group_in, item};
|
||||
|
||||
#[test]
|
||||
fn the_feedback_line_prints_exact_values() {
|
||||
let line = |last, head| EditFeedback::new(last, head).line();
|
||||
assert_eq!(
|
||||
EditFeedback::none().line(),
|
||||
"last edit: none; validation composite at head: none"
|
||||
);
|
||||
assert_eq!(
|
||||
line(Some(true), Some(0.5)),
|
||||
"last edit: accepted; validation composite at head: 0.500"
|
||||
);
|
||||
assert_eq!(
|
||||
line(Some(false), Some(0.0)),
|
||||
"last edit: rejected; validation composite at head: 0.000"
|
||||
);
|
||||
assert_eq!(line(None, None), EditFeedback::none().line());
|
||||
// The head is rounded to the nearest step before it is printed.
|
||||
assert_eq!(
|
||||
line(None, Some(0.5012)),
|
||||
"last edit: none; validation composite at head: 0.500"
|
||||
);
|
||||
assert_eq!(
|
||||
line(None, Some(0.5026)),
|
||||
"last edit: none; validation composite at head: 0.505"
|
||||
);
|
||||
assert_eq!(
|
||||
line(None, Some(0.9999)),
|
||||
"last edit: none; validation composite at head: 1.000"
|
||||
);
|
||||
assert_eq!(EditFeedback::new(None, Some(0.5026)).head, Some(0.505));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_offer_gate() {
|
||||
// A fresh checkpoint per call: this test is the gate, not the once rule.
|
||||
|
|
|
|||
|
|
@ -1,15 +1,23 @@
|
|||
//! The encoder step: what a model's answer becomes. This is the one place JSON
|
||||
//! is read (the xenos boundary): a model answers in JSON, `PacketDraft::parse`
|
||||
//! reads it, and nothing is stored as JSON. The model writes meaning only;
|
||||
//! `into_packet` sets the id, schema version, inventory revision, kind,
|
||||
//! The encoder and editor steps: what a model's answer becomes. This is the one
|
||||
//! place JSON is read (the xenos boundary): a model answers in JSON,
|
||||
//! `PacketDraft::parse` or `EditDraft::parse` reads it (both through
|
||||
//! `from_json`), and nothing is stored as JSON. The encoder writes meaning
|
||||
//! only; `into_packet` sets the id, schema version, inventory revision, kind,
|
||||
//! provenance, constraints and extensions, and an empty trace, so a model that
|
||||
//! saw a Validation item cannot hand its words to whoever reads packets.
|
||||
//! saw a Validation item cannot hand its words to whoever reads packets. The
|
||||
//! editor writes family, gloss, value kind and slots only; `into_revision` sets
|
||||
//! every id and revision number.
|
||||
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::corpus::CorpusItemId;
|
||||
use crate::inventory::{
|
||||
Change, Family, InvId, Inventory, InventoryEntry, InventoryRevision, Slot, ValueKind,
|
||||
};
|
||||
use crate::packet::{
|
||||
Constraints, Context, Discourse, EncoderId, Grade, Packet, PacketId, PacketKind, Predication,
|
||||
Provenance, ProvenanceSource, Referent, Relation, SCHEMA_VERSION, Stance, Trace,
|
||||
|
|
@ -58,12 +66,17 @@ impl fmt::Debug for DraftError {
|
|||
|
||||
impl std::error::Error for DraftError {}
|
||||
|
||||
/// The one JSON read of a model's answer.
|
||||
fn from_json<T: DeserializeOwned>(answer: &str) -> Result<T, DraftError> {
|
||||
serde_json::from_str(answer.trim()).map_err(|e| DraftError {
|
||||
line: e.line(),
|
||||
column: e.column(),
|
||||
})
|
||||
}
|
||||
|
||||
impl PacketDraft {
|
||||
pub fn parse(answer: &str) -> Result<Self, DraftError> {
|
||||
serde_json::from_str(answer.trim()).map_err(|e| DraftError {
|
||||
line: e.line(),
|
||||
column: e.column(),
|
||||
})
|
||||
from_json(answer)
|
||||
}
|
||||
|
||||
/// The draft that says what `packet` says.
|
||||
|
|
@ -105,6 +118,87 @@ impl PacketDraft {
|
|||
}
|
||||
}
|
||||
|
||||
/// The most changes (adds and retires together) one edit may make.
|
||||
pub const MAX_EDIT_CHANGES: usize = 8;
|
||||
|
||||
/// One entry an edit adds, as the editor writes it: no id and no revision.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EntryDraft {
|
||||
pub family: Family,
|
||||
pub gloss: String,
|
||||
#[serde(default)]
|
||||
pub value_kind: Option<ValueKind>,
|
||||
#[serde(default)]
|
||||
pub applies_to: Vec<Slot>,
|
||||
}
|
||||
|
||||
/// An inventory edit as the editor writes it: entries to add and ids to retire.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EditDraft {
|
||||
#[serde(default)]
|
||||
pub add: Vec<EntryDraft>,
|
||||
#[serde(default)]
|
||||
pub retire: Vec<InvId>,
|
||||
}
|
||||
|
||||
impl EditDraft {
|
||||
pub fn parse(answer: &str) -> Result<Self, DraftError> {
|
||||
from_json(answer)
|
||||
}
|
||||
|
||||
/// The revision after `base` this edit asks for, and the entries it adds.
|
||||
/// `None` when the edit makes no change or more than [`MAX_EDIT_CHANGES`].
|
||||
/// Every id is the store's: an added entry takes the next number of its
|
||||
/// family after the highest in `inventory`, retired entries included, in
|
||||
/// draft order. Nothing else is judged here: the gloss, the entry's shape,
|
||||
/// whether a retire target is live and a repeated id are
|
||||
/// `InventoryStore::admit_revision`'s to refuse.
|
||||
pub(crate) fn into_revision(
|
||||
self,
|
||||
inventory: &Inventory,
|
||||
base: u32,
|
||||
) -> Option<(InventoryRevision, Vec<InventoryEntry>)> {
|
||||
let count = self.add.len() + self.retire.len();
|
||||
if count == 0 || count > MAX_EDIT_CHANGES {
|
||||
return None;
|
||||
}
|
||||
let n = base.checked_add(1)?;
|
||||
let mut entries: Vec<InventoryEntry> = Vec::new();
|
||||
for draft in self.add {
|
||||
let highest = inventory
|
||||
.entries()
|
||||
.map(|e| e.id)
|
||||
.chain(entries.iter().map(|e| e.id))
|
||||
.filter(|id| id.family() == draft.family)
|
||||
.map(|id| id.n().get())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
entries.push(InventoryEntry {
|
||||
id: draft.family.id(NonZeroU32::new(highest.checked_add(1)?)?),
|
||||
gloss: draft.gloss,
|
||||
value_kind: draft.value_kind,
|
||||
applies_to: draft.applies_to,
|
||||
introduced_rev: n,
|
||||
retired_rev: None,
|
||||
});
|
||||
}
|
||||
let changes = entries
|
||||
.iter()
|
||||
.map(|e| Change::Add(e.id))
|
||||
.chain(self.retire.into_iter().map(Change::Retire))
|
||||
.collect();
|
||||
let revision = InventoryRevision {
|
||||
n,
|
||||
parent: Some(base),
|
||||
changes,
|
||||
frozen: false,
|
||||
};
|
||||
Some((revision, entries))
|
||||
}
|
||||
}
|
||||
|
||||
/// `encoder:<hex16>`, a digest of the model id and the prompt digest.
|
||||
pub fn encoder_id(model: &str, prompt_digest: &str) -> EncoderId {
|
||||
let digest = fnv1a64(format!("weksa-encoder-v1:{model}:{prompt_digest}").as_bytes());
|
||||
|
|
@ -228,4 +322,147 @@ mod tests {
|
|||
assert!(!text.contains("CANARY"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
fn entry(id: &str, retired: Option<u32>) -> InventoryEntry {
|
||||
let id = InvId::parse(id).unwrap();
|
||||
InventoryEntry {
|
||||
id,
|
||||
gloss: "g".into(),
|
||||
value_kind: (id.family() == Family::Extension).then_some(ValueKind::Flag),
|
||||
applies_to: vec![],
|
||||
introduced_rev: 1,
|
||||
retired_rev: retired,
|
||||
}
|
||||
}
|
||||
|
||||
fn add(family: Family, gloss: &str) -> EntryDraft {
|
||||
EntryDraft {
|
||||
family,
|
||||
gloss: gloss.into(),
|
||||
value_kind: None,
|
||||
applies_to: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_are_the_stores() {
|
||||
// concept:1 live, concept:3 retired (the highest), frame:2 live; no role.
|
||||
let inv = Inventory::new(
|
||||
vec![
|
||||
entry("concept:1", None),
|
||||
entry("concept:3", Some(2)),
|
||||
entry("frame:2", None),
|
||||
],
|
||||
vec![],
|
||||
);
|
||||
let draft = EditDraft {
|
||||
add: vec![
|
||||
add(Family::Concept, "first"),
|
||||
add(Family::Role, "role"),
|
||||
add(Family::Concept, "second"),
|
||||
add(Family::Frame, "frame"),
|
||||
],
|
||||
retire: vec![
|
||||
InvId::parse("concept:1").unwrap(),
|
||||
InvId::parse("frame:2").unwrap(),
|
||||
],
|
||||
};
|
||||
let (rev, entries) = draft.clone().into_revision(&inv, 6).unwrap();
|
||||
let ids: Vec<String> = entries.iter().map(|e| e.id.to_string()).collect();
|
||||
// After the highest of the family, retired ones included, in draft order.
|
||||
assert_eq!(ids, ["concept:4", "role:1", "concept:5", "frame:3"]);
|
||||
assert_eq!((rev.n, rev.parent, rev.frozen), (7, Some(6), false));
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.all(|e| e.introduced_rev == 7 && e.retired_rev.is_none())
|
||||
);
|
||||
let glosses: Vec<&str> = entries.iter().map(|e| e.gloss.as_str()).collect();
|
||||
assert_eq!(glosses, ["first", "role", "second", "frame"]);
|
||||
// Adds in draft order, then the retires in draft order.
|
||||
let changes: Vec<String> = rev
|
||||
.changes
|
||||
.iter()
|
||||
.map(|c| match c {
|
||||
Change::Add(id) => format!("+{id}"),
|
||||
Change::Retire(id) => format!("-{id}"),
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
changes,
|
||||
[
|
||||
"+concept:4",
|
||||
"+role:1",
|
||||
"+concept:5",
|
||||
"+frame:3",
|
||||
"-concept:1",
|
||||
"-frame:2"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_change_count_is_one_to_eight() {
|
||||
let inv = Inventory::new(vec![entry("concept:1", None)], vec![]);
|
||||
let adds = |n: usize| EditDraft {
|
||||
add: (0..n)
|
||||
.map(|i| add(Family::Concept, &format!("g{i}")))
|
||||
.collect(),
|
||||
retire: vec![],
|
||||
};
|
||||
assert!(adds(0).into_revision(&inv, 1).is_none());
|
||||
assert!(adds(1).into_revision(&inv, 1).is_some());
|
||||
assert_eq!(adds(8).into_revision(&inv, 1).unwrap().1.len(), 8);
|
||||
assert!(adds(9).into_revision(&inv, 1).is_none());
|
||||
// Adds and retires count together.
|
||||
let mixed = |a: usize, r: usize| EditDraft {
|
||||
add: (0..a).map(|_| add(Family::Concept, "g")).collect(),
|
||||
retire: vec![InvId::parse("concept:1").unwrap(); r],
|
||||
};
|
||||
assert!(mixed(4, 4).into_revision(&inv, 1).is_some());
|
||||
assert!(mixed(4, 5).into_revision(&inv, 1).is_none());
|
||||
assert!(mixed(0, 9).into_revision(&inv, 1).is_none());
|
||||
// A repeated retire is the door's to refuse, not this fn's.
|
||||
assert!(mixed(0, 2).into_revision(&inv, 1).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edit_draft_reads_what_an_editor_may_write() {
|
||||
let d = EditDraft::parse(
|
||||
r#" {"add":[{"family":"Feature","gloss":"g","value_kind":{"Options":["a","b"]},"applies_to":["Referent"]}],"retire":["concept:2"]} "#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(d.add[0].family, Family::Feature);
|
||||
assert_eq!(d.add[0].applies_to, [Slot::Referent]);
|
||||
assert_eq!(d.retire, [InvId::parse("concept:2").unwrap()]);
|
||||
// Both lists default; value_kind and applies_to default.
|
||||
let d = EditDraft::parse(r#"{"add":[{"family":"Concept","gloss":"g"}]}"#).unwrap();
|
||||
assert_eq!(d.add[0].value_kind, None);
|
||||
assert!(d.add[0].applies_to.is_empty() && d.retire.is_empty());
|
||||
assert_eq!(EditDraft::parse("{}").unwrap().add.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edit_draft_refuses_what_the_store_writes() {
|
||||
// An id, a revision number or a retired mark is the store's to write.
|
||||
for field in ["id", "introduced_rev", "retired_rev", "n", "parent"] {
|
||||
let json = format!(r#"{{"add":[{{"family":"Concept","gloss":"g","{field}":1}}]}}"#);
|
||||
assert!(EditDraft::parse(&json).is_err(), "{field} in an entry");
|
||||
}
|
||||
for field in ["revision", "frozen", "changes"] {
|
||||
let json = format!(r#"{{"add":[],"{field}":1}}"#);
|
||||
assert!(EditDraft::parse(&json).is_err(), "{field} in an edit");
|
||||
}
|
||||
// A family or a retire id that is not one.
|
||||
assert!(EditDraft::parse(r#"{"add":[{"family":"Nope","gloss":"g"}]}"#).is_err());
|
||||
assert!(EditDraft::parse(r#"{"retire":["concept:0"]}"#).is_err());
|
||||
assert!(EditDraft::parse(r#"{"retire":[1]}"#).is_err());
|
||||
// Not JSON, not an object, an error that names no token.
|
||||
let err = EditDraft::parse("{\n \"add\": CANARYTOKEN").unwrap_err();
|
||||
assert_eq!((err.line, err.column), (2, 10));
|
||||
for text in [err.to_string(), format!("{err:?}")] {
|
||||
assert!(!text.contains("CANARY"), "{text}");
|
||||
}
|
||||
assert!(EditDraft::parse("3").is_err() && EditDraft::parse("\"x\"").is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,6 +85,11 @@ impl InvId {
|
|||
pub fn family(self) -> Family {
|
||||
self.family
|
||||
}
|
||||
|
||||
/// The `n` of `<family prefix>:<n>`.
|
||||
pub fn n(self) -> NonZeroU32 {
|
||||
self.n
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for InvId {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ pub mod store;
|
|||
pub mod validate;
|
||||
|
||||
pub use corpus::{CorpusItem, CorpusItemId, NameSpan, Source, Turn};
|
||||
pub use encode::PacketDraft;
|
||||
pub use encode::{EditDraft, PacketDraft};
|
||||
pub use entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
|
||||
pub use form::FormError;
|
||||
pub use inventory::{
|
||||
|
|
@ -36,7 +36,7 @@ pub use packet::{
|
|||
EncoderId, EntityId, Grade, NodeId, Packet, PacketId, PacketKind, SCHEMA_VERSION, Stance,
|
||||
};
|
||||
pub use phones::{Phone, PhoneInventory, SegmentError, Stress};
|
||||
pub use run::{RoundRecord, RunStores, round};
|
||||
pub use run::{Decision, RoundRecord, RunStores, cycle, round};
|
||||
pub use score::{Embedder, ItemScore, OllamaEmbedder, Scorer, chrf};
|
||||
pub use speaker::{Annotator, PairRelation, Party, Scene, Speaker};
|
||||
pub use split::{Split, SplitScheme};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn};
|
|||
use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation};
|
||||
use weksa::inventory::{proof_revision, seed};
|
||||
use weksa::packet::proof_packet;
|
||||
use weksa::run::{RoundOutcome, RunPaths, RunStores, round};
|
||||
use weksa::run::{CycleOutcome, RoundOutcome, RunPaths, RunStores, cycle, round};
|
||||
use weksa::score::ScoreInput;
|
||||
use weksa::{
|
||||
AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore,
|
||||
|
|
@ -98,6 +98,15 @@ enum LoopCommand {
|
|||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
/// Run one edit-accept cycle of the run at ROOT. The stores are read only
|
||||
/// from the run's record. Prompts without answers are written to the
|
||||
/// cycle's directory and the command exits 3; rerun after the `.answer`
|
||||
/// files exist.
|
||||
Cycle {
|
||||
root: PathBuf,
|
||||
#[arg(long, default_value = "http://127.0.0.1:11434")]
|
||||
ollama_url: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
|
|
@ -758,6 +767,25 @@ fn main() -> Result<()> {
|
|||
RoundOutcome::Recorded(r) => println!("{}", r.summary_line()),
|
||||
}
|
||||
}
|
||||
Command::Loop(LoopCommand::Cycle { root, ollama_url }) => {
|
||||
let unreadable = |_| anyhow::anyhow!("a store would not read");
|
||||
let mut run = RunStores::open(&root)?;
|
||||
let mut files = AnswerFiles::new(
|
||||
run.next_cycle_dir().map_err(unreadable)?,
|
||||
run.record().model.clone(),
|
||||
);
|
||||
let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
|
||||
match cycle(&mut run, &mut files, &scorer)? {
|
||||
CycleOutcome::Unanswered { prompts } => {
|
||||
for digest in &prompts {
|
||||
println!("unanswered {digest}");
|
||||
}
|
||||
println!("unanswered {} prompts", prompts.len());
|
||||
std::process::exit(3);
|
||||
}
|
||||
CycleOutcome::Decided(decision) => println!("{}", decision.summary_line()),
|
||||
}
|
||||
}
|
||||
Command::Schema { out } => {
|
||||
std::fs::create_dir_all(&out)?;
|
||||
for (name, json) in weksa::schema::published() {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -5,7 +5,7 @@ use crate::corpus::CorpusItem;
|
|||
use crate::entity::Entity;
|
||||
use crate::inventory::{InventoryEntry, InventoryRevision};
|
||||
use crate::packet::Packet;
|
||||
use crate::run::{RoundRecord, RunRecord};
|
||||
use crate::run::{Decision, RoundRecord, RunRecord};
|
||||
use crate::speaker::{Scene, Speaker};
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use schemars::{JsonSchema, schema_for};
|
||||
|
|
@ -27,5 +27,6 @@ pub fn published() -> Vec<(String, String)> {
|
|||
published_for::<Scene>(),
|
||||
published_for::<RunRecord>(),
|
||||
published_for::<RoundRecord>(),
|
||||
published_for::<Decision>(),
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -309,6 +309,7 @@ fn schema_writes_the_published_files() {
|
|||
"weksa.interlingua_packet.schema.json",
|
||||
"weksa.inventory_entry.schema.json",
|
||||
"weksa.inventory_revision.schema.json",
|
||||
"weksa.loop_decision.schema.json",
|
||||
"weksa.loop_round.schema.json",
|
||||
"weksa.loop_run.schema.json",
|
||||
"weksa.scene.schema.json",
|
||||
|
|
@ -1407,6 +1408,102 @@ fn loop_open_and_unanswered_round() {
|
|||
);
|
||||
}
|
||||
|
||||
fn open_dao_run(d: &std::path::Path) {
|
||||
dao_stores(d);
|
||||
let o = weksa(
|
||||
d,
|
||||
&[
|
||||
"loop",
|
||||
"open",
|
||||
"canaryroot",
|
||||
"--label",
|
||||
"smoke",
|
||||
"--answerer",
|
||||
"fake-model",
|
||||
],
|
||||
);
|
||||
assert!(o.status.success(), "{}", out(&o));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_cycle_needs_a_floor() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
open_dao_run(d.path());
|
||||
let o = weksa(d.path(), &["loop", "cycle", "canaryroot"]);
|
||||
assert!(!o.status.success());
|
||||
let text = out(&o);
|
||||
assert!(
|
||||
text.contains("the run has no noise floor: record two rounds first"),
|
||||
"{text}"
|
||||
);
|
||||
for echoed in ["canaryroot", d.path().to_str().unwrap()] {
|
||||
assert!(!text.contains(echoed), "{text}");
|
||||
}
|
||||
assert!(!d.path().join("canaryroot/cycle-1").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_cycle_asks_then_decides() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
open_dao_run(d.path());
|
||||
// Two rounds of refusals: a floor of 0 and a base of 0.
|
||||
for n in 1..=2 {
|
||||
let first = weksa(d.path(), &["loop", "round", "canaryroot"]);
|
||||
assert_eq!(first.status.code(), Some(3), "{}", out(&first));
|
||||
for entry in std::fs::read_dir(d.path().join(format!("canaryroot/round-{n}"))).unwrap() {
|
||||
let path = entry.unwrap().path();
|
||||
std::fs::write(path.with_extension("answer"), "not json").unwrap();
|
||||
}
|
||||
let second = weksa(d.path(), &["loop", "round", "canaryroot"]);
|
||||
assert!(second.status.success(), "{}", out(&second));
|
||||
}
|
||||
|
||||
// The editor is asked first: one digest, a count, exit 3, the prompt on disk.
|
||||
let ask = weksa(d.path(), &["loop", "cycle", "canaryroot"]);
|
||||
assert_eq!(ask.status.code(), Some(3), "{}", out(&ask));
|
||||
let text = out(&ask);
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
assert_eq!(lines.len(), 2, "{text}");
|
||||
let digest = lines[0].strip_prefix("unanswered ").unwrap();
|
||||
assert_eq!(digest.len(), 16, "{text}");
|
||||
assert_eq!(lines[1], "unanswered 1 prompts");
|
||||
for echoed in ["canaryroot", d.path().to_str().unwrap()] {
|
||||
assert!(!text.contains(echoed), "{text}");
|
||||
}
|
||||
let dir = d.path().join("canaryroot/cycle-1");
|
||||
let prompt = std::fs::read_to_string(dir.join(format!("{digest}.prompt"))).unwrap();
|
||||
assert!(prompt.starts_with(weksa::context::MINT_INSTRUCTIONS));
|
||||
|
||||
// An answer that is not an edit closes the cycle as a rejection, with no
|
||||
// embed call and no echo of the answer.
|
||||
std::fs::write(dir.join(format!("{digest}.answer")), "CANARYANSWER").unwrap();
|
||||
let done = weksa(d.path(), &["loop", "cycle", "canaryroot"]);
|
||||
assert!(done.status.success(), "{}", out(&done));
|
||||
assert_eq!(
|
||||
out(&done),
|
||||
"cycle 1 rev 3->none base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse\n"
|
||||
);
|
||||
|
||||
// The next cycle asks afresh in its own directory.
|
||||
let next = weksa(d.path(), &["loop", "cycle", "canaryroot"]);
|
||||
assert_eq!(next.status.code(), Some(3), "{}", out(&next));
|
||||
let text = out(&next);
|
||||
assert!(text.ends_with("unanswered 1 prompts\n"), "{text}");
|
||||
assert!(!text.to_uppercase().contains("CANARY"), "{text}");
|
||||
let asked = text
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap()
|
||||
.strip_prefix("unanswered ")
|
||||
.unwrap();
|
||||
assert_ne!(asked, digest);
|
||||
assert!(
|
||||
d.path()
|
||||
.join(format!("canaryroot/cycle-2/{asked}.prompt"))
|
||||
.is_file()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loop_errors_echo_no_input() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ mod common;
|
|||
|
||||
use common::{cast, ent, entities_in, group_in_bucket, item_in, model, run_over, world};
|
||||
use weksa::context::{
|
||||
ENCODE_INSTRUCTIONS as ENCODE, MINT_INSTRUCTIONS as MINT, OfferRefusal, Prompt,
|
||||
ENCODE_INSTRUCTIONS as ENCODE, EditFeedback, MINT_INSTRUCTIONS as MINT, OfferRefusal, Prompt,
|
||||
};
|
||||
use weksa::run::RunStores;
|
||||
use weksa::{
|
||||
|
|
@ -111,17 +111,23 @@ fn train_items_are_in_key_order() {
|
|||
);
|
||||
assert_eq!((t.len(), t.is_empty()), (3, false));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let none = run_over(dir.path()).train().unwrap();
|
||||
let empty = run_over(dir.path());
|
||||
let none = empty.train().unwrap();
|
||||
assert_eq!((none.len(), none.is_empty()), (0, true));
|
||||
assert_eq!(Prompt::mint(&none).text(), MINT);
|
||||
assert!(Prompt::mint(&none).item_keys().is_empty());
|
||||
let (inv, fb) = (empty.inventory().inventory().unwrap(), EditFeedback::none());
|
||||
assert_eq!(
|
||||
Prompt::mint(&none, &inv, 0, &fb).text(),
|
||||
format!("{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0")
|
||||
);
|
||||
assert!(Prompt::mint(&none, &inv, 0, &fb).item_keys().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mint_prompt_holds_train_text_only() {
|
||||
let c = corpus();
|
||||
let t = c.run.train().unwrap();
|
||||
let p = Prompt::mint(&t);
|
||||
let inv = c.run.inventory().inventory().unwrap();
|
||||
let p = Prompt::mint(&t, &inv, 4, &EditFeedback::none());
|
||||
assert!(p.text().starts_with(MINT));
|
||||
assert!(p.text().contains("alphamarker"));
|
||||
for other in ["betamarker", "gammamarker", "deltamarker", "epsilonmarker"] {
|
||||
|
|
@ -149,16 +155,20 @@ fn prompts_render_deterministically() {
|
|||
.unwrap();
|
||||
let run = run_over(dir.path());
|
||||
let t = run.train().unwrap();
|
||||
let inv = run.inventory().inventory().unwrap();
|
||||
let fb = EditFeedback::none();
|
||||
assert_eq!(
|
||||
Prompt::mint(&t).text(),
|
||||
Prompt::mint(&t, &inv, 0, &fb).text(),
|
||||
format!(
|
||||
"{MINT}\n\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line."
|
||||
"{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0\n\n[item:a:1] entity:nibu: Hello there.\n\n[item:a:2] entity:nibu: Second line."
|
||||
)
|
||||
);
|
||||
assert_eq!(Prompt::mint(&t).item_keys(), ["item:a:1", "item:a:2"]);
|
||||
assert_eq!(
|
||||
Prompt::mint(&t, &inv, 0, &fb).item_keys(),
|
||||
["item:a:1", "item:a:2"]
|
||||
);
|
||||
|
||||
let offer = run.offer("item:a:2").unwrap().unwrap();
|
||||
let inv = run.inventory().inventory().unwrap();
|
||||
assert_eq!(
|
||||
Prompt::encode(&offer, &t, &inv, 0).text(),
|
||||
format!(
|
||||
|
|
@ -375,7 +385,8 @@ fn prior_turns_print_only_text_a_live_train_item_holds() {
|
|||
|
||||
let run = run_over(dir.path());
|
||||
let t = run.train().unwrap();
|
||||
let mint = Prompt::mint(&t);
|
||||
let inv = run.inventory().inventory().unwrap();
|
||||
let mint = Prompt::mint(&t, &inv, 0, &EditFeedback::none());
|
||||
for secret in ["withdrawnsecret", "testsecret", "validation reply"] {
|
||||
assert!(!mint.text().contains(secret), "mint printed {secret}");
|
||||
}
|
||||
|
|
@ -384,7 +395,6 @@ fn prior_turns_print_only_text_a_live_train_item_holds() {
|
|||
"mint prints no prior turns"
|
||||
);
|
||||
|
||||
let inv = run.inventory().inventory().unwrap();
|
||||
let encode = |key: &str| {
|
||||
Prompt::encode(&run.offer(key).unwrap().unwrap(), &t, &inv, 0)
|
||||
.text()
|
||||
|
|
@ -504,3 +514,28 @@ fn prior_turn_fold_matches_the_store() {
|
|||
}
|
||||
|
||||
const ELIDED_MARK: &str = "[elided]";
|
||||
|
||||
#[test]
|
||||
fn mint_and_encode_list_the_same_inventory() {
|
||||
let w = world();
|
||||
let run = run_over(w.dir.path());
|
||||
let inv = run.inventory().inventory().unwrap();
|
||||
let t = run.train().unwrap();
|
||||
let offer = run.offer("item:nibu:1").unwrap().unwrap();
|
||||
let fb = EditFeedback::none();
|
||||
for rev in [2, 4] {
|
||||
let header = format!("inventory rev:{rev}\n");
|
||||
let mint = Prompt::mint(&t, &inv, rev, &fb);
|
||||
let encode = Prompt::encode(&offer, &t, &inv, rev);
|
||||
let from_mint = mint.text().split_once(&header).unwrap().1;
|
||||
let from_mint = from_mint.split("\n\n").next().unwrap();
|
||||
let from_encode = encode.text().split_once(&header).unwrap().1;
|
||||
assert!(!from_mint.is_empty());
|
||||
assert_eq!(from_mint, from_encode, "rev {rev}");
|
||||
}
|
||||
let at = |rev| {
|
||||
let text = Prompt::mint(&t, &inv, rev, &fb).text().to_string();
|
||||
text.split_once("inventory rev:").unwrap().1.to_string()
|
||||
};
|
||||
assert_ne!(at(2), at(4), "the revision selects the entries");
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -134,6 +134,12 @@ fn published_schemas_match() {
|
|||
let published = weksa::schema::published();
|
||||
let mut names: Vec<String> = published.iter().map(|(n, _)| n.clone()).collect();
|
||||
names.sort();
|
||||
for loop_type in ["run", "round", "decision"] {
|
||||
assert!(
|
||||
names.contains(&format!("weksa.loop_{loop_type}.schema.json")),
|
||||
"{loop_type}"
|
||||
);
|
||||
}
|
||||
let mut on_disk: Vec<String> = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||||
|
|
|
|||
|
|
@ -40,8 +40,9 @@ means no store holds it and nothing writes it.
|
|||
| Scene (cut `speaker-scene`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker |
|
||||
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
|
||||
| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid |
|
||||
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, proposed change, validation scores before and after, outcome | Opened, scored, closed accepted or rejected; immutable once closed. Lives in the run store on Yggdrasil | The loop driver's accept rule (validation composite improves past the measured noise floor and no metric regresses past tolerance) |
|
||||
| Score | Key `<lowering-key>:<metric>@<metric-version>`; type `weksa.score` | Immutable; a new metric version scores again under a new key | The scorer, deterministic given the metric version and the embedder's model id. Cut `scorer` computes scores as values; cut `golf-loop-en` stores them, because the lowering keys they hang from arrive with the loop |
|
||||
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, Validation means and refusal counts, the digest of the inventory file it was scored under and, for a cycle's candidate round, its cycle `k` | Scored and recorded whole; immutable. A candidate round is put in one batch with its Decision. Lives in the run store on Yggdrasil | `run::round` and `run::cycle` (both through `score_round`) |
|
||||
| Loop decision | Key `run:<label>:cycle:<k>`; type `weksa.loop_decision` with the base round and revision, the edit prompt's digest and Train keys, the editor's answer digest, the candidate revision, inventory digest and round, the base, candidate and floor readings (chrF, meaning, composite), the threshold and the verdict (Accepted, or Rejected for edit-parse, edit-admit, no-gain, regressed chrF or meaning); digests and numbers, no text | Immutable. The latest Accepted one names the run's head inventory (`<root>/cycle-<k>/inventory.cc`, its candidate digest, its candidate round); with none the head is the copy `create` made at `<root>/inventory.cc`. Lives in the run store on Yggdrasil | `run::cycle`'s accept rule, `run::decide`: the validation composite gains more than max(noise floor, `LADDER_STEP`) and no metric falls past max(its floor, one step), differences compared in billionths so a gain of exactly one step is a tie |
|
||||
| Score | Key `<lowering-key>:<metric>@<metric-version>`; type `weksa.score` | Immutable; a new metric version scores again under a new key | The scorer, deterministic given the metric version and the embedder's model id. Cut `scorer` computes scores as values; cut `golf-loop-en` stores them, because the lowering keys they hang from arrive with the loop. Lands with follow-up `golf-loop-score-docs`, in the round's directory; nothing in the cycle reads them |
|
||||
| Lowering trace | Key `lowering:<packet-key>:<profile>@<rev>:<lowerer-digest>:<sample>`; type `weksa.lowering` holding the surface text and its trace together | Immutable. A surface text cannot exist without its trace, because they are one record (`lowering-traced`) | The lowerer emits; the lowerer sees the packet without its `trace` (the input type has no such field), never the corpus item |
|
||||
| Conlang lexicon entry | Key `lex:<language>:<concept-id>`; root form, morphology class | Root placed at its concept's embedding when the concept first needs a form in that language; stable once accepted; diachrony derives later forms, it does not overwrite | The language project's owner: the operator for her settings' languages, the loop proposing roots for her review |
|
||||
| Conlang grammar and phonology data | `<language>/grammar`, `<language>/phonology` documents per language project | Revised by commit; every lowering names the revision | The language project's owner |
|
||||
|
|
@ -405,7 +406,8 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
|
|||
6. `pd-importer`: public-domain play and fiction dialogue into the corpus with speaker, scene group, edition provenance, and Speaker and Scene annotations by the ruled annotator. Waits on 4, follow-up `gutenberg-terms` and question `pd-speaker-annotation`.
|
||||
7a. `loop-context-builder`: `context.rs` (`TrainItems`, `EncoderOffer`, `Prompt`: corpus text reaches a model call only through `Prompt::mint` over live Train items or `Prompt::encode` over one offered item; instructions are the consts `MINT_INSTRUCTIONS` and `ENCODE_INSTRUCTIONS` in `context.rs`; `Checkpoint` has no public constructor, offers each item once, and its offers borrow it (the gate is typed `TestReadings` and Test offers on the round path; operator tools such as `weksa corpus dump` and `weksa score` still read corpus text of any split); prior turns print the held Train text), `SplitScheme::live`, `CorpusItem::cited_entities` (one owner of the id set `PacketStore::admit` and the offer share), and checkpoint-only Test readings (`Checkpoint`, `TestReadings`; `Aggregate` and `leak_diagnostic` carry no Test). Library only; since 7b `TrainItems::from_corpus` and `EncoderOffer::for_round` are crate-private and `RunStores` is their one public producer. Merges follow-ups `loop-context-builder-2` and `scorer-test-readings-type`. (spec r1)
|
||||
7b. `golf-loop-round` (spec r1; branch `weksa-lowering/golf-loop-round`): the smallest loop that scores one round. `run.rs`: `RunStores` pins the corpus and entity stores of a run by canonical path and FNV-1a 64 digest at `create`, reopens only from `<root>/run.cc` and refuses drift (corpus, entities, `PROMPT_SOURCE_DIGEST` of `context.rs`, `LOWERER_SOURCE_DIGEST` of `lower.rs`, model, embedder); `round` encodes every live Train and Validation item through a `ModelPort` (`model.rs`; the one adapter `AnswerFiles` writes `<digest>.prompt` and reads `<digest>.answer`, so the model is the Claude session), reads each answer into a `PacketDraft` (`encode.rs`, the one place a model's JSON is read), admits through `PacketStore::admit` into `<root>/round-<n>/packets.cc`, lowers, scores once, and keeps a `RoundRecord` (`weksa.loop_round`, no text, no Test; Train rows, Validation only as means and refusal counts). `weksa loop open|round`. Not here: the spec-edit step, accept rule and composite, noise floor, Score and lowering documents, a subprocess adapter, the checkpoint door, a model lowerer (follow-ups `golf-loop-spec-edit`, `golf-loop-accept-rule`, `golf-loop-score-documents`, `golf-loop-subprocess-adapter`, `golf-loop-checkpoint-door-bound`, `golf-loop-model-lowerer`). `packet-store-inventory-binding` is met for loop packets only (`RunStores` admits with the run's inventory); `validate`, `packet proof|dao` and `lower` still take caller stores. Waits on 7a, `checkpoint-gate`, 4b.
|
||||
7. `golf-loop-en` (7b landed its round, `ModelPort`, encoder prompt, template digest and round records; the rest remains): the loop driver, the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, lowerer prompts, Score documents, the composite and accept rule, `weksa score`, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
7c. `golf-loop-edit-accept` (spec r1; branch `weksa-lowering/golf-loop-edit-accept`): one edit-accept cycle. The run works on its own copy of the inventory (`create` copies the caller's file to `<root>/inventory.cc`, refusing a root that already holds a file of that name, and pins its digest as `RunRecord` key 11; `RunRecord.inventory` is provenance; a `run.cc` from before has no key 11 and does not open); the head is derived from the latest Accepted `Decision`. `weksa loop cycle` asks the editor for one inventory edit from `Prompt::mint(train, inventory, rev, feedback)` (Train blocks, the inventory listing and an `EditFeedback` line: whether the last edit was kept and the head's Validation composite rounded to `LADDER_STEP`, the Ladder release of Blum and Hardt 2015), reads it as an `EditDraft` (`encode.rs`; ids and revisions are the store's, `into_revision`), admits it through `InventoryStore::admit_revision` into `<root>/cycle-<k>/inventory.cc`, runs a candidate round under it, and `decide`s on Validation against the head's establishing round past a floor measured as the spread between rounds 1 and 2 (never under one `LADDER_STEP`). An edit that fails is a rejected `Decision`, not an error. `RoundRecord` gains keys 16 (inventory digest) and 17 (cycle); an encode answer that is not UTF-8 is that item's parse refusal. Not here: Score and lowering documents (`golf-loop-score-docs`), Train rows in the edit prompt (`golf-loop-edit-train-feedback`), a paired floor (`golf-loop-paired-floor`), the checkpoint door and merging to `data/` (`golf-loop-checkpoint-door-bound`, `golf-loop-freeze-merge`), a model lowerer (`golf-loop-model-lowerer-next`; until it lands an entry an edit adds has no lowering row, so a cycle gains only by retiring entries or adding ones the encoder does not cite). Waits on 7b.
|
||||
7. `golf-loop-en` (7b and 7c landed the round, `ModelPort`, encoder prompt, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, lowerer prompts, Score documents, `weksa score` in the loop, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3.
|
||||
9. `ja-panel`: ja-JP into the loop's panel. Waits on 7.
|
||||
10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`.
|
||||
|
|
|
|||
201
schemas/cultnet/weksa.loop_decision.schema.json
Normal file
201
schemas/cultnet/weksa.loop_decision.schema.json
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Decision",
|
||||
"description": "One closed edit-accept cycle: digests and numbers, no text. Key\n`run:<label>:cycle:<k>`. Only [`cycle`] builds one.\n\n```\nfn f(d: &weksa::run::Decision) -> u32 { d.k() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin is\nthe same literal in the unit test `a_decision_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{Call, Decision, Reading, Verdict};\nfn f() -> Decision {\n let r = Reading { chrf: 0.0, meaning: 0.0, composite: 0.0 };\n Decision {\n k: 1,\n base_round: 1,\n base_rev: 1,\n base_inventory_digest: String::new(),\n edit: Call { prompt: String::new(), items: vec![] },\n edit_answer_digest: None,\n candidate_rev: None,\n candidate_inventory_digest: None,\n candidate_round: None,\n base: r,\n candidate: None,\n floor: r,\n threshold: 0.0,\n verdict: Verdict::Accepted,\n }\n}\n```",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"base": {
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
"base_inventory_digest": {
|
||||
"type": "string"
|
||||
},
|
||||
"base_rev": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"base_round": {
|
||||
"description": "The head's establishing round: the base of this cycle.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"candidate": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"candidate_inventory_digest": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"candidate_rev": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"candidate_round": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"edit": {
|
||||
"description": "The edit prompt's digest and the Train keys it held.",
|
||||
"$ref": "#/$defs/Call"
|
||||
},
|
||||
"edit_answer_digest": {
|
||||
"description": "16 hex of the editor's answer; `None` when it was unreadable.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"floor": {
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
"k": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"threshold": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"verdict": {
|
||||
"$ref": "#/$defs/Verdict"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"k",
|
||||
"base_round",
|
||||
"base_rev",
|
||||
"base_inventory_digest",
|
||||
"edit",
|
||||
"base",
|
||||
"floor",
|
||||
"threshold",
|
||||
"verdict"
|
||||
],
|
||||
"$defs": {
|
||||
"Call": {
|
||||
"description": "One model call: the prompt's digest and the item keys it held.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"prompt",
|
||||
"items"
|
||||
]
|
||||
},
|
||||
"Metric": {
|
||||
"description": "Which metric an edit regressed.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Chrf",
|
||||
"Meaning"
|
||||
]
|
||||
},
|
||||
"Reading": {
|
||||
"description": "One validation reading: chrF 0 to 100, meaning 0 to 1, and their composite.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"chrf": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"composite": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"meaning": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"chrf",
|
||||
"meaning",
|
||||
"composite"
|
||||
]
|
||||
},
|
||||
"Verdict": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Accepted"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Rejected": {
|
||||
"$ref": "#/$defs/Why"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Rejected"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"Why": {
|
||||
"description": "Why an edit was rejected.",
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"NoGain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "The answer was not an edit draft.",
|
||||
"type": "string",
|
||||
"const": "EditParse"
|
||||
},
|
||||
{
|
||||
"description": "The draft asked for no change or too many, or the inventory door refused it.",
|
||||
"type": "string",
|
||||
"const": "EditAdmit"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"Regressed": {
|
||||
"$ref": "#/$defs/Metric"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"Regressed"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "RoundRecord",
|
||||
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n }\n}\n```",
|
||||
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n lowerer_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n }\n}\n```",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"calls": {
|
||||
|
|
@ -16,12 +16,25 @@
|
|||
"corpus_digest": {
|
||||
"type": "string"
|
||||
},
|
||||
"cycle": {
|
||||
"description": "`Some(k)` for the candidate round of cycle `k`.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"embedder": {
|
||||
"type": "string"
|
||||
},
|
||||
"entity_digest": {
|
||||
"type": "string"
|
||||
},
|
||||
"inventory_digest": {
|
||||
"description": "16 hex of the inventory file the round was scored under.",
|
||||
"type": "string"
|
||||
},
|
||||
"inventory_rev": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
|
|
@ -85,7 +98,8 @@
|
|||
"validation_mean",
|
||||
"validation_refused",
|
||||
"train_scored",
|
||||
"validation_scored"
|
||||
"validation_scored",
|
||||
"inventory_digest"
|
||||
],
|
||||
"$defs": {
|
||||
"Call": {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,11 @@
|
|||
"type": "string"
|
||||
},
|
||||
"inventory": {
|
||||
"description": "Canonical absolute path of the inventory store the run was opened over:\nprovenance only. `create` copies that file to `<root>/inventory.cc` and\nthe run reads nothing at this path afterwards.",
|
||||
"type": "string"
|
||||
},
|
||||
"inventory_digest": {
|
||||
"description": "16 hex of the bytes `create` copied to `<root>/inventory.cc`.",
|
||||
"type": "string"
|
||||
},
|
||||
"label": {
|
||||
|
|
@ -53,6 +58,7 @@
|
|||
"lowerer_digest",
|
||||
"model",
|
||||
"embedder",
|
||||
"chrf_version"
|
||||
"chrf_version",
|
||||
"inventory_digest"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue