ja-panel-loop: language is the item's, the panel the run's; panel accept rule #49
15 changed files with 1595 additions and 475 deletions
|
|
@ -8,7 +8,9 @@
|
|||
//! consts in this module; every prompt also lists the inventory at the run's
|
||||
//! revision. A mint prompt is the editor's: it reads [`TrainItems`], an
|
||||
//! [`Inventory`] and an [`EditFeedback`] and nothing else, so no held-out text,
|
||||
//! key or per-item score can reach it.
|
||||
//! key or per-item score can reach it. A lowering prompt names the language it
|
||||
//! is said in on a language line; every language is named by its tag, none by a
|
||||
//! default.
|
||||
//!
|
||||
//! Item text may hold line breaks; the renderer indents every continuation line
|
||||
//! (`indented`), so item text cannot forge a `[key]` line or a paragraph.
|
||||
|
|
@ -25,7 +27,7 @@ use serde::Serialize;
|
|||
|
||||
use crate::corpus::{CorpusItem, CorpusItemId};
|
||||
use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind};
|
||||
use crate::lower::{EntityForms, LowerError, TARGET_LANGUAGE, entity};
|
||||
use crate::lower::{EntityForms, LowerError, entity};
|
||||
use crate::packet::{
|
||||
Context, Discourse, EntityId, Extension, Predication, Referent, Relation, Stance,
|
||||
};
|
||||
|
|
@ -37,7 +39,7 @@ use crate::validate::LoweringInput;
|
|||
/// The instructions a mint prompt starts with: the editor's whole brief. Fixed
|
||||
/// per run by `PROMPT_SOURCE_DIGEST`.
|
||||
pub const MINT_INSTRUCTIONS: &str = "\
|
||||
Propose one edit to the inventory below, so that the lines below can be encoded and lowered to English more faithfully.
|
||||
Propose one edit to the inventory below, so that the lines below can be encoded and lowered, each in its own language, more faithfully.
|
||||
Answer with one JSON object and nothing else: no code fence, no commentary.
|
||||
Shape: {\"add\": [{\"family\", \"gloss\", \"value_kind\", \"applies_to\"}], \"retire\": [ids]}; 1 to 8 changes in all, adds and retires together.
|
||||
- family: one of Concept, Frame, Role, Relation, SpeechAct, Feature, ReferentKind, Extension.
|
||||
|
|
@ -65,7 +67,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab
|
|||
/// The instructions a lowering prompt starts with: the lowerer's whole brief.
|
||||
/// Fixed per run by `PROMPT_SOURCE_DIGEST`.
|
||||
pub const LOWER_INSTRUCTIONS: &str = "\
|
||||
Say the packet below in English (en-US), as one utterance of its speaker: all of its meaning and nothing it does not carry.
|
||||
Say the packet below in the language on the language line, as one utterance of its speaker: all of its meaning and nothing it does not carry.
|
||||
Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says.
|
||||
A referent with an entity is named by one of that entity's forms on the entities line.
|
||||
Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary.";
|
||||
|
|
@ -81,14 +83,14 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
|
|||
/// reads; it cannot be built from a slice, and no accessor returns text or an
|
||||
/// item.
|
||||
///
|
||||
/// ```
|
||||
/// ```no_run
|
||||
/// # let dir = tempfile::tempdir().unwrap();
|
||||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// # let panel = ["en-US".to_string()];
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// assert!(t.is_empty());
|
||||
/// let _ = t.len();
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
|
|
@ -100,7 +102,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let _u = t.clone();
|
||||
/// ```
|
||||
|
|
@ -111,7 +113,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// s(&t);
|
||||
/// ```
|
||||
|
|
@ -120,11 +122,14 @@ pub struct TrainItems {
|
|||
}
|
||||
|
||||
impl TrainItems {
|
||||
pub(crate) fn from_corpus(corpus: &CorpusStore) -> anyhow::Result<Self> {
|
||||
/// The live Train items whose language is in `panel`.
|
||||
pub(crate) fn from_corpus(corpus: &CorpusStore, panel: &[String]) -> anyhow::Result<Self> {
|
||||
let items = corpus
|
||||
.all()?
|
||||
.into_iter()
|
||||
.filter(|item| SplitScheme::V1.live(item) == Some(Split::Train))
|
||||
.filter(|item| {
|
||||
SplitScheme::V1.live(item) == Some(Split::Train) && panel.contains(&item.language)
|
||||
})
|
||||
.collect();
|
||||
Ok(Self { items })
|
||||
}
|
||||
|
|
@ -252,11 +257,13 @@ impl<'cp> EncoderOffer<'cp> {
|
|||
}
|
||||
|
||||
/// What the editor is told of how its last edit fared: whether it was kept, and
|
||||
/// the held-out composite at the current inventory, rounded to the nearest
|
||||
/// [`LADDER_STEP`]. The head composite changes only when an edit is accepted and
|
||||
/// an accept needs a gain above one step, so a run releases at most
|
||||
/// `1 / LADDER_STEP` distinct values (Blum and Hardt 2015, the Ladder). Nothing
|
||||
/// else about the held-out split is a field.
|
||||
/// the held-out composite at the current inventory of each panel language,
|
||||
/// rounded to the nearest [`LADDER_STEP`]. The heads change only when an edit is
|
||||
/// accepted, and an accept raises the panel's composite sum by more than one
|
||||
/// step, so a run over `L` languages releases at most `1.5 * L / LADDER_STEP`
|
||||
/// changes of this line (Blum and Hardt 2015, the Ladder; ruling
|
||||
/// `weksa-lowering:ruling:panel-accept-sum-rule`). Nothing else about the
|
||||
/// held-out split is a field.
|
||||
///
|
||||
/// ```
|
||||
/// let f = weksa::context::EditFeedback::none();
|
||||
|
|
@ -266,18 +273,18 @@ impl<'cp> EncoderOffer<'cp> {
|
|||
/// Its fields are private:
|
||||
///
|
||||
/// ```compile_fail,E0451
|
||||
/// let _ = weksa::context::EditFeedback { last: None, head: None };
|
||||
/// let _ = weksa::context::EditFeedback { last: None, head: vec![] };
|
||||
/// ```
|
||||
///
|
||||
/// and only the run builds one with a reading:
|
||||
///
|
||||
/// ```compile_fail,E0624
|
||||
/// let _ = weksa::context::EditFeedback::new(Some(true), Some(0.5));
|
||||
/// let _ = weksa::context::EditFeedback::new(Some(true), vec![]);
|
||||
/// ```
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct EditFeedback {
|
||||
last: Option<bool>,
|
||||
head: Option<f64>,
|
||||
head: Vec<(String, f64)>,
|
||||
}
|
||||
|
||||
impl EditFeedback {
|
||||
|
|
@ -285,16 +292,20 @@ impl EditFeedback {
|
|||
pub fn none() -> Self {
|
||||
Self {
|
||||
last: None,
|
||||
head: None,
|
||||
head: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `last`: whether the previous edit was accepted; `head`: the composite at
|
||||
/// the current inventory, rounded here to the nearest [`LADDER_STEP`].
|
||||
pub(crate) fn new(last: Option<bool>, head: Option<f64>) -> Self {
|
||||
/// `last`: whether the previous edit was accepted; `head`: each panel
|
||||
/// language's composite at the current inventory, in panel order, rounded
|
||||
/// here to the nearest [`LADDER_STEP`].
|
||||
pub(crate) fn new(last: Option<bool>, head: Vec<(String, f64)>) -> Self {
|
||||
Self {
|
||||
last,
|
||||
head: head.map(|x| (x / LADDER_STEP).round() * LADDER_STEP),
|
||||
head: head
|
||||
.into_iter()
|
||||
.map(|(tag, x)| (tag, (x / LADDER_STEP).round() * LADDER_STEP))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -304,7 +315,15 @@ impl EditFeedback {
|
|||
Some(false) => "rejected",
|
||||
None => "none",
|
||||
};
|
||||
let head = self.head.map_or("none".to_string(), |x| format!("{x:.3}"));
|
||||
let head = if self.head.is_empty() {
|
||||
"none".to_string()
|
||||
} else {
|
||||
self.head
|
||||
.iter()
|
||||
.map(|(tag, x)| format!("{tag} {x:.3}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
};
|
||||
format!("last edit: {last}; validation composite at head: {head}")
|
||||
}
|
||||
}
|
||||
|
|
@ -312,12 +331,12 @@ impl EditFeedback {
|
|||
/// Text for a model call, and the keys of the items it holds. The only
|
||||
/// constructors are [`Prompt::mint`], [`Prompt::encode`] and [`Prompt::lower`].
|
||||
///
|
||||
/// ```
|
||||
/// ```no_run
|
||||
/// # let dir = tempfile::tempdir().unwrap();
|
||||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// # let panel = ["en-US".to_string()];
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
|
|
@ -335,7 +354,7 @@ impl EditFeedback {
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
|
|
@ -348,7 +367,7 @@ impl EditFeedback {
|
|||
/// # let p = |n: &str| dir.path().join(n);
|
||||
/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); }
|
||||
/// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") };
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap();
|
||||
/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap();
|
||||
/// let t = run.train().unwrap();
|
||||
/// let inv = run.inventory().inventory().unwrap();
|
||||
/// let feedback = weksa::context::EditFeedback::none();
|
||||
|
|
@ -502,17 +521,19 @@ impl Prompt {
|
|||
}
|
||||
}
|
||||
|
||||
/// [`LOWER_INSTRUCTIONS`], the meaning of `input` (no kind, constraints,
|
||||
/// scene, speaker or group), the en-US forms of every entity its referents
|
||||
/// cite, and the entries of `inventory` live at the view's revision. `item`
|
||||
/// [`LOWER_INSTRUCTIONS`], the `language: <tag>` line, the meaning of
|
||||
/// `input` (no kind, constraints, scene, speaker or group), the forms in
|
||||
/// `language` of every entity its referents cite, and the entries of `inventory` live at the view's revision. `item`
|
||||
/// goes only to [`Prompt::item_keys`], never into the text. The view's
|
||||
/// speaker and group fields are never read; a speaker's entity id appears
|
||||
/// only when the encoder cites that entity on a referent, which is naming a
|
||||
/// cited entity like any other. A cited entity the port lacks, or one with
|
||||
/// no en-US form, is an error and no prompt.
|
||||
/// no form in `language`, is an error and no prompt. The caller chooses the
|
||||
/// language; the view has none.
|
||||
pub fn lower(
|
||||
input: &LoweringInput,
|
||||
item: &CorpusItemId,
|
||||
language: &str,
|
||||
inventory: &Inventory,
|
||||
entities: &dyn EntityForms,
|
||||
) -> Result<Prompt, LowerError> {
|
||||
|
|
@ -520,7 +541,7 @@ impl Prompt {
|
|||
for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) {
|
||||
let found = entity(entities, id)?;
|
||||
let named = found
|
||||
.forms_in(TARGET_LANGUAGE)
|
||||
.forms_in(language)
|
||||
.filter(|f| !f.is_empty())
|
||||
.ok_or_else(|| LowerError::NoForm { entity: id.clone() })?;
|
||||
forms.insert(id.as_str(), named.to_vec());
|
||||
|
|
@ -536,7 +557,7 @@ impl Prompt {
|
|||
};
|
||||
let rev = input.inventory_rev();
|
||||
let text = format!(
|
||||
"{LOWER_INSTRUCTIONS}\n\npacket: {}\n\nentities: {}\n\ninventory rev:{rev}{}",
|
||||
"{LOWER_INSTRUCTIONS}\n\nlanguage: {language}\n\npacket: {}\n\nentities: {}\n\ninventory rev:{rev}{}",
|
||||
serde_json::to_string(&meaning).expect("a meaning is plain data"),
|
||||
serde_json::to_string(&forms).expect("forms are plain data"),
|
||||
inventory_lines(inventory, rev)
|
||||
|
|
@ -583,34 +604,50 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn the_feedback_line_prints_exact_values() {
|
||||
let en = |x: f64| vec![("en-US".to_string(), x)];
|
||||
let line = |last, head| EditFeedback::new(last, head).line();
|
||||
assert_eq!(
|
||||
EditFeedback::none().line(),
|
||||
"last edit: none; validation composite at head: none"
|
||||
);
|
||||
assert_eq!(
|
||||
line(Some(true), Some(0.5)),
|
||||
"last edit: accepted; validation composite at head: 0.500"
|
||||
line(Some(true), en(0.5)),
|
||||
"last edit: accepted; validation composite at head: en-US 0.500"
|
||||
);
|
||||
assert_eq!(
|
||||
line(Some(false), Some(0.0)),
|
||||
"last edit: rejected; validation composite at head: 0.000"
|
||||
line(Some(false), en(0.0)),
|
||||
"last edit: rejected; validation composite at head: en-US 0.000"
|
||||
);
|
||||
assert_eq!(line(None, None), EditFeedback::none().line());
|
||||
assert_eq!(line(None, vec![]), EditFeedback::none().line());
|
||||
// The head is rounded to the nearest step before it is printed.
|
||||
assert_eq!(
|
||||
line(None, Some(0.5012)),
|
||||
"last edit: none; validation composite at head: 0.500"
|
||||
line(None, en(0.5012)),
|
||||
"last edit: none; validation composite at head: en-US 0.500"
|
||||
);
|
||||
assert_eq!(
|
||||
line(None, Some(0.5026)),
|
||||
"last edit: none; validation composite at head: 0.505"
|
||||
line(None, en(0.5026)),
|
||||
"last edit: none; validation composite at head: en-US 0.505"
|
||||
);
|
||||
assert_eq!(
|
||||
line(None, Some(0.9999)),
|
||||
"last edit: none; validation composite at head: 1.000"
|
||||
line(None, en(0.9999)),
|
||||
"last edit: none; validation composite at head: en-US 1.000"
|
||||
);
|
||||
assert_eq!(EditFeedback::new(None, Some(0.5026)).head, Some(0.505));
|
||||
// One value per panel language, in the order given.
|
||||
assert_eq!(
|
||||
line(None, vec![("en-US".into(), 0.5026), ("ja-JP".into(), 0.41)]),
|
||||
"last edit: none; validation composite at head: en-US 0.505, ja-JP 0.410"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mint_prompt_over_no_train_items_is_the_brief_the_feedback_and_the_listing() {
|
||||
let none = TrainItems { items: vec![] };
|
||||
let prompt = Prompt::mint(&none, &Inventory::default(), 0, &EditFeedback::none());
|
||||
assert_eq!(
|
||||
prompt.text(),
|
||||
format!("{MINT_INSTRUCTIONS}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0")
|
||||
);
|
||||
assert!(prompt.item_keys().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -43,8 +43,10 @@ use crate::phones::{Phone, PhoneInventory, SegmentError};
|
|||
use crate::store::EntityStore;
|
||||
use crate::validate::LoweringInput;
|
||||
|
||||
/// The language tag this lowerer's entity forms are read under, spelled as an
|
||||
/// entity's `LanguageForms::language` and a corpus item's language are.
|
||||
/// The language tag of the template table and the phones: what `weksa lower`
|
||||
/// says in. The golf loop lowers each item in its own language and does not
|
||||
/// read this. Spelled as an entity's `LanguageForms::language` and a corpus
|
||||
/// item's language are.
|
||||
pub const TARGET_LANGUAGE: &str = "en-US";
|
||||
|
||||
/// The lowerer's whole view of the entity store: one entity by id. Narrow so a
|
||||
|
|
|
|||
|
|
@ -99,6 +99,10 @@ enum LoopCommand {
|
|||
/// that family wrote. Defaults to unrecorded.
|
||||
#[arg(long, default_value = "unrecorded")]
|
||||
answerer_family: ModelFamilyArg,
|
||||
/// The languages the run offers, reads and decides on: BCP 47 tags,
|
||||
/// comma-separated. Every one needs a live Train and Validation item.
|
||||
#[arg(long, value_delimiter = ',', required = true)]
|
||||
panel: Vec<String>,
|
||||
},
|
||||
/// Run one round of the run at ROOT. The stores are read only from the
|
||||
/// run's record: the global --corpus, --inventory and --entities are not
|
||||
|
|
@ -329,7 +333,8 @@ struct AddArgs {
|
|||
/// NFC-normalized here; `--name` byte offsets index the normalized text.
|
||||
#[arg(long)]
|
||||
text: String,
|
||||
#[arg(long, default_value = "en-US")]
|
||||
/// The item's language, a BCP 47 tag; no language is the default.
|
||||
#[arg(long)]
|
||||
language: String,
|
||||
/// An entity id.
|
||||
#[arg(long)]
|
||||
|
|
@ -765,6 +770,7 @@ fn main() -> Result<()> {
|
|||
embed_model,
|
||||
lowerer_reach,
|
||||
answerer_family,
|
||||
panel,
|
||||
}) => {
|
||||
let run = RunStores::create(
|
||||
&root,
|
||||
|
|
@ -778,6 +784,7 @@ fn main() -> Result<()> {
|
|||
&embed_model,
|
||||
lowerer_reach,
|
||||
answerer_family.into(),
|
||||
&panel,
|
||||
)?;
|
||||
println!("opened run:{}", run.record().label);
|
||||
}
|
||||
|
|
@ -821,12 +828,12 @@ fn main() -> Result<()> {
|
|||
std::process::exit(3);
|
||||
}
|
||||
AblationOutcome::Read(readings) => {
|
||||
for (split, reading) in readings {
|
||||
for (language, split, reading) in readings {
|
||||
let name = match split {
|
||||
Split::Train => "train",
|
||||
_ => "validation",
|
||||
};
|
||||
println!("{name} {}", reading.line());
|
||||
println!("{language} {name} {}", reading.line());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -13,8 +13,10 @@
|
|||
//! word, never one word per character: per-character words would make every
|
||||
//! honest single-word gloss share bigrams with a held-out sentence, which is
|
||||
//! the leak signature. A change between such a run and any other script also
|
||||
//! ends a word. A leak in these scripts shows at n = 1 and in chrF; a
|
||||
//! dictionary segmenter is the ja-JP panel's to add.
|
||||
//! ends a word. The n >= 2 coverage signature is therefore blind on these
|
||||
//! scripts: a leak there shows at n = 1 and in max pairwise chrF, which is
|
||||
//! character n-grams and script-blind. A dictionary segmenter is follow-up
|
||||
//! `ja-word-segmenter`.
|
||||
|
||||
use caseless::default_case_fold_str;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ fn out(o: &Output) -> String {
|
|||
String::from_utf8_lossy(&o.stdout).into_owned() + &String::from_utf8_lossy(&o.stderr)
|
||||
}
|
||||
|
||||
const ADD: [&str; 14] = [
|
||||
const ADD: [&str; 16] = [
|
||||
"corpus",
|
||||
"add",
|
||||
"--slug",
|
||||
|
|
@ -62,6 +62,8 @@ const ADD: [&str; 14] = [
|
|||
"2026-10-29",
|
||||
"--source",
|
||||
"cast-authored",
|
||||
"--language",
|
||||
"en-US",
|
||||
];
|
||||
|
||||
#[test]
|
||||
|
|
@ -149,6 +151,8 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
|
|||
let base = [
|
||||
"corpus",
|
||||
"add",
|
||||
"--language",
|
||||
"en-US",
|
||||
"--slug",
|
||||
"x",
|
||||
"--speaker",
|
||||
|
|
@ -375,6 +379,8 @@ fn entity_put_and_name_spans() {
|
|||
&[
|
||||
"corpus",
|
||||
"add",
|
||||
"--language",
|
||||
"en-US",
|
||||
"--slug",
|
||||
"kay",
|
||||
"--text",
|
||||
|
|
@ -547,6 +553,8 @@ fn add_with(dir: &std::path::Path, source: &str, extra: &[&str]) -> Output {
|
|||
let mut args = vec![
|
||||
"corpus",
|
||||
"add",
|
||||
"--language",
|
||||
"en-US",
|
||||
"--slug",
|
||||
"nibu",
|
||||
"--text",
|
||||
|
|
@ -903,6 +911,8 @@ fn proof_packet_lowers_and_the_lowering_scores_against_items() {
|
|||
&[
|
||||
"corpus",
|
||||
"add",
|
||||
"--language",
|
||||
"en-US",
|
||||
"--slug",
|
||||
"nibu",
|
||||
"--text",
|
||||
|
|
@ -1062,7 +1072,7 @@ fn cli_errors_never_echo_identifiers() {
|
|||
let p = d.path();
|
||||
cast_entities(p, &["nibu"]);
|
||||
let add = |extra: &[&str]| -> Vec<String> {
|
||||
["corpus", "add", "--admitted-on", "2026-10-29"]
|
||||
["corpus", "add", "--language", "en-US", "--admitted-on", "2026-10-29"]
|
||||
.into_iter()
|
||||
.chain(extra.iter().copied())
|
||||
.map(String::from)
|
||||
|
|
@ -1344,6 +1354,8 @@ fn loop_open_and_unanswered_round() {
|
|||
"smoke",
|
||||
"--answerer",
|
||||
"fake-model",
|
||||
"--panel",
|
||||
"en-US",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
"--answerer-family",
|
||||
|
|
@ -1397,7 +1409,7 @@ fn loop_open_and_unanswered_round() {
|
|||
assert!(second.status.success(), "{}", out(&second));
|
||||
assert_eq!(
|
||||
out(&second),
|
||||
"round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n"
|
||||
"round 1 rev 3 en-US train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n"
|
||||
);
|
||||
|
||||
// The global store flags are not read by loop round: a drifted store is refused.
|
||||
|
|
@ -1424,6 +1436,8 @@ fn open_dao_run(d: &std::path::Path) {
|
|||
"smoke",
|
||||
"--answerer",
|
||||
"fake-model",
|
||||
"--panel",
|
||||
"en-US",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
],
|
||||
|
|
@ -1441,6 +1455,8 @@ fn pinned_family(d: &std::path::Path, root: &str, flag: &[&str]) -> weksa::corpu
|
|||
"fam",
|
||||
"--answerer",
|
||||
"fake-model",
|
||||
"--panel",
|
||||
"en-US",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
];
|
||||
|
|
@ -1526,7 +1542,7 @@ fn loop_cycle_asks_then_decides() {
|
|||
assert!(done.status.success(), "{}", out(&done));
|
||||
assert_eq!(
|
||||
out(&done),
|
||||
"cycle 1 rev 3->none base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n"
|
||||
"cycle 1 rev 3->none en-US base=0.0000 candidate=none floor=0.0000 threshold=0.0050 rejected edit-parse lowering=tool-less-relay\n"
|
||||
);
|
||||
|
||||
// The next cycle asks afresh in its own directory.
|
||||
|
|
@ -1565,6 +1581,8 @@ fn loop_errors_echo_no_input() {
|
|||
"CANARYLABEL",
|
||||
"--answerer",
|
||||
"m",
|
||||
"--panel",
|
||||
"en-US",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
],
|
||||
|
|
@ -1576,6 +1594,8 @@ fn loop_errors_echo_no_input() {
|
|||
"ok",
|
||||
"--answerer",
|
||||
"CANARY\n",
|
||||
"--panel",
|
||||
"en-US",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
],
|
||||
|
|
@ -1597,6 +1617,8 @@ fn loop_errors_echo_no_input() {
|
|||
"ok",
|
||||
"--answerer",
|
||||
"m",
|
||||
"--panel",
|
||||
"en-US",
|
||||
"--lowerer-reach",
|
||||
"tool-less-relay",
|
||||
])
|
||||
|
|
@ -1610,7 +1632,7 @@ fn loop_errors_echo_no_input() {
|
|||
fn loop_open_requires_the_lowerer_reach_and_pins_it() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
dao_stores(d.path());
|
||||
let base = ["loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m"];
|
||||
let base = ["loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m", "--panel", "en-US"];
|
||||
// No default: the record says how lowerings are answered, or there is no run.
|
||||
let o = weksa(d.path(), &base);
|
||||
assert!(!o.status.success());
|
||||
|
|
@ -1698,7 +1720,7 @@ fn loop_ablate_waits_then_prints_one_line_per_split() {
|
|||
let text = out(&second);
|
||||
assert_eq!(
|
||||
text,
|
||||
"train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nvalidation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n"
|
||||
"en-US train ablation n=1 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=1 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=0 chrf=none meaning=none moved=0 ref_gain_chrf=none ref_gain_meaning=none\nen-US validation ablation n=2 unusable=0 replay chrf=100.00 meaning=1.000 | drop n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000 | permute n=2 chrf=100.00 meaning=1.000 moved=0 ref_gain_chrf=0.00 ref_gain_meaning=0.000\n"
|
||||
);
|
||||
for leaked in ["item:", "packet:", "highest excellence", "canaryroot"] {
|
||||
assert!(!text.contains(leaked), "{text}");
|
||||
|
|
@ -1720,3 +1742,24 @@ fn loop_ablate_without_the_rounds_answers_says_so() {
|
|||
assert!(text.contains("holds no answer") && text.contains("head inventory"), "{text}");
|
||||
assert!(!text.contains("would not read"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn panel_and_language_are_required() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
dao_stores(d.path());
|
||||
// No panel: a run says what it reads, or there is no run.
|
||||
let o = weksa(
|
||||
d.path(),
|
||||
&[
|
||||
"loop", "open", "canaryroot", "--label", "smoke", "--answerer", "m",
|
||||
"--lowerer-reach", "tool-less-relay",
|
||||
],
|
||||
);
|
||||
assert!(!o.status.success());
|
||||
assert!(out(&o).contains("--panel"), "{}", out(&o));
|
||||
assert!(!d.path().join("canaryroot/run.cc").exists());
|
||||
// No language: no language is the unmarked one.
|
||||
let o = weksa(d.path(), &ADD[..ADD.len() - 2]);
|
||||
assert!(!o.status.success());
|
||||
assert!(out(&o).contains("--language"), "{}", out(&o));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,12 +116,30 @@ pub fn entities_in(dir: &std::path::Path) -> EntityStore {
|
|||
|
||||
/// A run over the stores `world()` and `corpus()` keep in `dir`: `c.cc`,
|
||||
/// `i.cc` and `entities.cc`. Read at the call, so create it after the items.
|
||||
/// The run's en-US panel needs a live Train and a live Validation item: a
|
||||
/// corpus short of one gets `item:panel:<n>` added, and no other item.
|
||||
pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
|
||||
for name in ["c.cc", "i.cc", "entities.cc"] {
|
||||
if !dir.join(name).exists() {
|
||||
std::fs::write(dir.join(name), [0x90]).unwrap();
|
||||
}
|
||||
}
|
||||
let entities = entities_in(dir);
|
||||
let mut corpus = CorpusStore::open(&dir.join("c.cc")).unwrap();
|
||||
for (n, split) in [(1, weksa::Split::Train), (2, weksa::Split::Validation)] {
|
||||
let held = corpus
|
||||
.all()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|i| SplitScheme::V1.live(i) == Some(split));
|
||||
if !held {
|
||||
let group = group_in_bucket(if n == 1 { 10 } else { 50 });
|
||||
let key = format!("item:panel:{n}");
|
||||
let text = format!("Panel fixture line {n}.");
|
||||
corpus.add(item_in(&key, &text, &group, cast()), &entities).unwrap();
|
||||
}
|
||||
}
|
||||
drop(corpus);
|
||||
weksa::run::RunStores::create(
|
||||
&dir.join("run"),
|
||||
"t",
|
||||
|
|
@ -134,6 +152,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores {
|
|||
"bag",
|
||||
weksa::run::LowererReach::ToolLessRelay,
|
||||
weksa::corpus::ModelFamily::Unrecorded,
|
||||
&["en-US".to_string()],
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
|
@ -219,6 +238,10 @@ pub fn world() -> World {
|
|||
end: 4,
|
||||
}];
|
||||
corpus.add(first, &entities).unwrap();
|
||||
// A run's panel needs a live Validation item as well as a Train one.
|
||||
let validation = group_in_bucket(50);
|
||||
let second = item_in("item:panel:1", "Nibu says watch the ladder.", &validation, cast());
|
||||
corpus.add(second, &entities).unwrap();
|
||||
let mut inventory = InventoryStore::open(&dir.path().join("i.cc")).unwrap();
|
||||
let (rev1, entries) = seed();
|
||||
inventory.admit_revision(rev1, entries).unwrap();
|
||||
|
|
|
|||
|
|
@ -110,16 +110,6 @@ fn train_items_are_in_key_order() {
|
|||
["item:a:1", "item:m:1", "item:z:1"]
|
||||
);
|
||||
assert_eq!((t.len(), t.is_empty()), (3, false));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let empty = run_over(dir.path());
|
||||
let none = empty.train().unwrap();
|
||||
assert_eq!((none.len(), none.is_empty()), (0, true));
|
||||
let (inv, fb) = (empty.inventory().inventory().unwrap(), EditFeedback::none());
|
||||
assert_eq!(
|
||||
Prompt::mint(&none, &inv, 0, &fb).text(),
|
||||
format!("{MINT}\n\nlast edit: none; validation composite at head: none\n\ninventory rev:0")
|
||||
);
|
||||
assert!(Prompt::mint(&none, &inv, 0, &fb).item_keys().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -606,13 +596,28 @@ mod lowering {
|
|||
}
|
||||
}
|
||||
|
||||
/// The language line carries the tag the caller passes; no instruction
|
||||
/// names a language.
|
||||
#[test]
|
||||
fn lower_prompt_names_its_language() {
|
||||
let (view, inv, entities, item) = dao(2);
|
||||
for tag in ["en-US", "ja-JP", "pt-BR"] {
|
||||
let prompt = Prompt::lower(&view, &item, tag, &inv, &entities).unwrap();
|
||||
let want = format!("{LOWER}\n\nlanguage: {tag}\n\npacket: ");
|
||||
assert!(prompt.text().starts_with(&want), "{tag}");
|
||||
}
|
||||
for brief in [LOWER, weksa::context::MINT_INSTRUCTIONS] {
|
||||
assert!(!brief.contains("English") && !brief.contains("en-US"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_prompt_text_is_pinned() {
|
||||
let (view, inv, entities, item) = dao(2);
|
||||
let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap();
|
||||
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap();
|
||||
let (head, listing) = prompt.text().split_once("inventory rev:3\n").unwrap();
|
||||
let expected = format!(
|
||||
"{LOWER}\n\npacket: {}\n\nentities: {{}}\n\n",
|
||||
"{LOWER}\n\nlanguage: en-US\n\npacket: {}\n\nentities: {{}}\n\n",
|
||||
concat!(
|
||||
r#"{"context":{"setting":null,"medium":null,"participants":[],"audience":[]},"#,
|
||||
r#""discourse":{"speech_act":"speech_act:1","addressee":null,"addressee_consent":null,"features":[]},"#,
|
||||
|
|
@ -628,7 +633,7 @@ mod lowering {
|
|||
assert!(listing.contains("\nfeature:18: "));
|
||||
// The brief says what the spec says it must.
|
||||
for needed in [
|
||||
"en-US",
|
||||
"the language on the language line",
|
||||
"all of its meaning and nothing it does not carry",
|
||||
"a grade is permille, 0 to 1000",
|
||||
"felt affect and restraint",
|
||||
|
|
@ -671,7 +676,7 @@ mod lowering {
|
|||
let mut packet = packets().remove(2);
|
||||
packet.stance = stance.clone();
|
||||
let view = admitted(packet, &store, "daodejing/corpus.cc", &entities);
|
||||
let prompt = Prompt::lower(&view, &item, &inv, &entities).unwrap();
|
||||
let prompt = Prompt::lower(&view, &item, "en-US", &inv, &entities).unwrap();
|
||||
(stance, prompt)
|
||||
};
|
||||
let three = [prompt_of(own), prompt_of(angry), prompt_of(Stance::default())];
|
||||
|
|
@ -708,7 +713,7 @@ mod lowering {
|
|||
("sa-Latn", &["Nibuwa"]),
|
||||
("en-US", &["Nibu", "Nibu's"]),
|
||||
])]);
|
||||
let prompt = Prompt::lower(&view, &item, &cast.inventory().unwrap(), &held).unwrap();
|
||||
let prompt = Prompt::lower(&view, &item, "en-US", &cast.inventory().unwrap(), &held).unwrap();
|
||||
assert!(
|
||||
prompt
|
||||
.text()
|
||||
|
|
@ -738,7 +743,7 @@ mod lowering {
|
|||
),
|
||||
(Held(vec![]), LowerError::UnknownEntity { entity: id }),
|
||||
] {
|
||||
assert_eq!(Prompt::lower(&view, &item, &inv, &held).err(), Some(want));
|
||||
assert_eq!(Prompt::lower(&view, &item, "en-US", &inv, &held).err(), Some(want));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -238,12 +238,14 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
|
|||
"bag",
|
||||
weksa::run::LowererReach::BriefedSubagent,
|
||||
ModelFamily::Unrecorded,
|
||||
&["en-US".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
let mut model = Unrelated(vec![]);
|
||||
let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else {
|
||||
let RecallOutcome::Read(reading) = run.recall(&mut model).unwrap() else {
|
||||
panic!("answered");
|
||||
};
|
||||
let r = reading.languages[0].clone();
|
||||
assert_eq!((r.n, r.answered), (28, 28));
|
||||
assert!(r.chrf < 25.0, "{}", r.chrf);
|
||||
assert_eq!(model.0.len(), 28);
|
||||
|
|
@ -251,9 +253,12 @@ fn the_recall_probe_covers_every_live_original_item_and_names_no_work() {
|
|||
let all = CorpusStore::open(&c).unwrap().all().unwrap();
|
||||
for item in &all {
|
||||
let live = SplitScheme::V1.live(item);
|
||||
// A recall prompt shows the first half of the words.
|
||||
let words: Vec<&str> = item.text.split_whitespace().collect();
|
||||
let opening = words[..words.len() / 2].join(" ");
|
||||
// A recall prompt shows the first half of the item, cut at a word boundary.
|
||||
let mut cut = item.text.len() / 4;
|
||||
while !item.text.is_char_boundary(cut) {
|
||||
cut -= 1;
|
||||
}
|
||||
let opening = item.text[..cut].trim_end().to_string();
|
||||
let shown = model.0.iter().any(|t| t.contains(&opening));
|
||||
assert_eq!(
|
||||
shown,
|
||||
|
|
@ -309,6 +314,7 @@ fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::Ro
|
|||
"bag",
|
||||
LowererReach::ToolLessRelay,
|
||||
family,
|
||||
&["en-US".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
let scorer = weksa::score::Scorer::new(Bag);
|
||||
|
|
@ -416,9 +422,10 @@ fn recall_probes_only_live_train_and_validation_items() {
|
|||
drop(store);
|
||||
let run = run_over(dir.path());
|
||||
let mut m = Record(vec![]);
|
||||
let RecallOutcome::Read(r) = run.recall(&mut m).unwrap() else {
|
||||
let RecallOutcome::Read(reading) = run.recall(&mut m).unwrap() else {
|
||||
panic!("answered");
|
||||
};
|
||||
let r = reading.languages[0].clone();
|
||||
assert_eq!(r.n, 2);
|
||||
let shown = |mark: &str| m.0.iter().any(|t| t.contains(mark));
|
||||
assert!(shown("trainmark") && shown("validmark"));
|
||||
|
|
@ -464,6 +471,7 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
|
|||
"bag",
|
||||
LowererReach::ToolLessRelay,
|
||||
ModelFamily::Unrecorded,
|
||||
&["en-US".to_string()],
|
||||
)
|
||||
.unwrap();
|
||||
let items = CorpusStore::open(&p("corpus.cc")).unwrap().all().unwrap();
|
||||
|
|
@ -499,9 +507,10 @@ fn no_test_item_is_offered_to_a_round_or_recall_or_recorded() {
|
|||
assert!(r.train().iter().all(|row| row.item != item.id));
|
||||
}
|
||||
let mut recall = Record(vec![]);
|
||||
let weksa::run::RecallOutcome::Read(rr) = run.recall(&mut recall).unwrap() else {
|
||||
let weksa::run::RecallOutcome::Read(reading) = run.recall(&mut recall).unwrap() else {
|
||||
panic!("answered");
|
||||
};
|
||||
let rr = reading.languages[0].clone();
|
||||
assert_eq!((rr.n, rr.answered), (28, 28));
|
||||
for item in &held {
|
||||
assert!(!leaks(&recall.0, item), "recall prompt held {}", item.id);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ plus a separately recorded chosen gloss per lowering).
|
|||
is applied anywhere both meet.
|
||||
- Rebuild: `weksa --corpus C --inventory I --entities E` with `inventory seed`,
|
||||
`inventory proof`, `inventory dao`, `entity put entity:laozi ...`,
|
||||
`corpus add ... --source public-domain` per item, then `packet dao <store>`.
|
||||
`corpus add ... --language en-US --source public-domain` per item, then `packet dao <store>`.
|
||||
- Source: Project Gutenberg eBook 216, "The Tao Teh King, or the Tao and its
|
||||
Characteristics", Laozi, tr. James Legge (Sacred Books of the East 39, 1891).
|
||||
Gutenberg's licence (1.E.2) lets a work derived from texts not protected by US
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ means no store holds it and nothing writes it.
|
|||
| Scene (cut `speaker-scene`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision, and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split | As Speaker |
|
||||
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
|
||||
| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid |
|
||||
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, Validation means and refusal counts, the digest of the inventory file it was scored under and, for a cycle's candidate round, its cycle `k` | Scored and recorded whole; immutable. A candidate round is put in one batch with its Decision. Lives in the run store on Yggdrasil | `run::round` and `run::cycle` (both through `score_round`) |
|
||||
| Loop decision | Key `run:<label>:cycle:<k>`; type `weksa.loop_decision` with the base round and revision, the edit prompt's digest and Train keys, the editor's answer digest, the candidate revision, inventory digest and round, the base, candidate and floor readings (chrF, meaning, composite), the threshold and the verdict (Accepted, or Rejected for edit-parse, edit-admit, no-gain, regressed chrF or meaning); digests and numbers, no text | Immutable. The latest Accepted one names the run's head inventory (`<root>/cycle-<k>/inventory.cc`, its candidate digest, its candidate round); with none the head is the copy `create` made at `<root>/inventory.cc`. Lives in the run store on Yggdrasil | `run::cycle`'s accept rule, `run::decide`: the validation composite gains more than max(noise floor, `LADDER_STEP`) and no metric falls past max(its floor, one step), differences compared in billionths so a gain of exactly one step is a tie |
|
||||
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, Validation means and refusal counts (from cut `ja-panel-loop`: one set per language of the run's panel, no pooled mean), the digest of the inventory file it was scored under and, for a cycle's candidate round, its cycle `k` | Scored and recorded whole; immutable. A candidate round is put in one batch with its Decision. Lives in the run store on Yggdrasil | `run::round` and `run::cycle` (both through `score_round`) |
|
||||
| Loop decision | Key `run:<label>:cycle:<k>`; type `weksa.loop_decision` with the base round and revision, the edit prompt's digest and Train keys, the editor's answer digest, the candidate revision, inventory digest and round, the base, candidate and floor readings (chrF, meaning, composite), the threshold and the verdict (Accepted, or Rejected for edit-parse, edit-admit, no-gain, regressed chrF, meaning or composite); digests and numbers, no text | Immutable. The latest Accepted one names the run's head inventory (`<root>/cycle-<k>/inventory.cc`, its candidate digest, its candidate round); with none the head is the copy `create` made at `<root>/inventory.cc`. Lives in the run store on Yggdrasil | `run::cycle`'s accept rule, `run::decide`: the validation composite gains more than max(noise floor, `LADDER_STEP`), no metric falls past max(its floor, one step), the composite does not fall past max(its floor, one step), and a non-finite reading rejects (the one regression test, `run::regression`); differences compared in billionths so a gain of exactly one step is a tie. From cut `ja-panel-loop` the readings, floor and threshold are kept per panel language and `run::regression` runs per language and is the one regression test (`decide` adds the gain test for a one-language panel): any language regressing rejects (verdicts `regressed-chrf`, `regressed-meaning`, `regressed-composite`), otherwise a one-language panel accepts as `decide` does and a panel of two or more accepts when the sum of composite deltas exceeds one `LADDER_STEP` (billionths). Given up: strict Pareto non-regression; a language may lose up to its own noise floor if the panel nets more than a step, because zero tolerance ratchets every base to a favourable noise draw and strangles the loop. Termination: each accept raises the composite sum by more than a step and composites lie in -0.5 to 1, so a run accepts at most 1.5 * languages / `LADDER_STEP` edits |
|
||||
| Score | Key `<lowering-key>:<metric>@<metric-version>`; type `weksa.score` | Immutable; a new metric version scores again under a new key | The scorer, deterministic given the metric version and the embedder's model id. Cut `scorer` computes scores as values; cut `golf-loop-en` stores them, because the lowering keys they hang from arrive with the loop. Lands with follow-up `golf-loop-score-docs`, in the round's directory; nothing in the cycle reads them |
|
||||
| Lowering trace | Key `lowering:<packet-key>:<profile>@<rev>:<lowerer-digest>:<sample>`; type `weksa.lowering` holding the surface text and its trace together | Immutable. A surface text cannot exist without its trace, because they are one record (`lowering-traced`) | The lowerer emits; the lowerer sees the packet without its `trace` (the input type has no such field), never the corpus item |
|
||||
| Conlang lexicon entry | Key `lex:<language>:<concept-id>`; root form, morphology class | Root placed at its concept's embedding when the concept first needs a form in that language; stable once accepted; diachrony derives later forms, it does not overwrite | The language project's owner: the operator for her settings' languages, the loop proposing roots for her review |
|
||||
|
|
@ -411,7 +411,7 @@ One line per cut; specs exist for the cuts about to run (2d and 4, siblings on `
|
|||
7d. `golf-loop-model-lowerer` (spec r1; branch `weksa-lowering/golf-loop-model-lowerer`): the round lowers through the `ModelPort`. `Prompt::lower` (`context.rs`) takes a `LoweringInput` (no text, key, group or speaker getter), the inventory at the view's revision and the en-US entity forms, and prints `LOWER_INSTRUCTIONS`, the meaning of the packet, the entities line and the inventory listing; the item key goes only to `item_keys`. The answer is `{"text"}`, read as `LoweringDraft` in `encode.rs`, held in `score_round`'s locals and scored once; it is never put in `run.cc`, a `RoundRecord`, a `Decision` or `EditFeedback`. A round has two answer phases (every encode prompt, then every lowering prompt), all or nothing: while any prompt is unanswered `packets.cc` is discarded and nothing reaches `run.cc`. `lower::lower` and the template table leave the loop (they remain `weksa lower`'s; follow-up `table-retirement`), so an entry an edit adds can be said and a cycle can buy a gain. One source pin remains: `PROMPT_SOURCE_DIGEST` covers `context.rs`, `split.rs` and `lower.rs` (`RunRecord` key 7 and `RoundRecord` key 3 are retired). The run's record pins `LowererReach` (`weksa loop open --lowerer-reach briefed-subagent|tool-less-relay`, `RunRecord` key 12) and every printed round and cycle line says it; a briefed-subagent run says its numbers are smoke-round only (rulings `lowerer-call-reach-toolless-relay`, `dao-round-seed-inventory`). A lowering is refused at `Lower` when blank, over `MAX_LOWERING_BYTES` (4096) or holding a control character other than a line break or tab; `PROMPT_SOURCE_DIGEST` also covers `packet.rs`. Not here: per-item lowering text documents (`golf-loop-score-docs`), the subprocess adapter. Waits on 7c.
|
||||
7. `golf-loop-en` (7b and 7c landed the round, `ModelPort`, encoder prompt, template digest, round records and the edit-accept cycle; the rest remains): the packet store bound to one inventory store everywhere (`packet-store-inventory-binding`), a subprocess adapter, Score documents, `weksa score` in the loop, the detached unit on Yggdrasil. Waits on 3, 5, 6 and follow-up `claude-on-yggdrasil`.
|
||||
8. `style-axis`: a style-embedding score behind the scorer port. Waits on 3.
|
||||
9. `ja-panel`: ja-JP into the loop's panel. Waits on 7.
|
||||
9. `ja-panel`: ja-JP into the loop's panel, in two cuts on `e9cbd9b`. 9a `ja-panel-loop` (spec r1): language is the item's and the panel the run's: `Prompt::lower` takes the item's language and prints it on a language line (no prompt names English), entity forms are read in that language, `RunRecord` pins the panel (`weksa loop open --panel`), offers, mint and recall read only panel items, round means, floors and readings are per panel language, the accept rule runs `decide` per language, the recall split works on unspaced scripts, and `weksa corpus add` has no default language. 9b `ja-panel-corpus` (spec r1, after 9a): the ja-JP slice of `data/original`, model-written Japanese Train and Validation passages and eight Aozora Bunko public-domain Test sentences with full provenance, and their entities. Cut 5 is not a prerequisite: neither language lowers with ontology entries until cut 5 gives both theirs.
|
||||
10. `conlang-lexicon`: the first conlang's lexicon placed at concept embeddings, its morphology data, its ontology entries and the gloss reader. Waits on 3, 5 and question `first-conlang`.
|
||||
11. `convergence-freeze`: the convergence test, `Checkpoint`'s one door (built from the convergence window over the persisted round records), the freeze, the re-encode into training pairs. Waits on 7, 9 and 10.
|
||||
12. `distil`: fine-tune a lowerer on the frozen training set. Waits on 11 and a host question asked then.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "Decision",
|
||||
"description": "One closed edit-accept cycle: digests and numbers, no text. Key\n`run:<label>:cycle:<k>`. Only [`cycle`] builds one.\n\n```\nfn f(d: &weksa::run::Decision) -> u32 { d.k() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin is\nthe same literal in the unit test `a_decision_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{Call, Decision, Reading, Verdict};\nfn f() -> Decision {\n let r = Reading { chrf: 0.0, meaning: 0.0, composite: 0.0 };\n Decision {\n k: 1,\n base_round: 1,\n base_rev: 1,\n base_inventory_digest: String::new(),\n edit: Call { prompt: String::new(), items: vec![] },\n edit_answer_digest: None,\n candidate_rev: None,\n candidate_inventory_digest: None,\n candidate_round: None,\n base: r,\n candidate: None,\n floor: r,\n threshold: 0.0,\n verdict: Verdict::Accepted,\n }\n}\n```",
|
||||
"description": "One closed edit-accept cycle: digests and numbers, no text. Key\n`run:<label>:cycle:<k>`. Only [`cycle`] builds one.\n\n```\nfn f(d: &weksa::run::Decision) -> u32 { d.k() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin is\nthe same literal in the unit test `a_decision_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{Call, Decision, Verdict};\nfn f() -> Decision {\n Decision {\n k: 1,\n base_round: 1,\n base_rev: 1,\n base_inventory_digest: String::new(),\n edit: Call { prompt: String::new(), items: vec![] },\n edit_answer_digest: None,\n candidate_rev: None,\n candidate_inventory_digest: None,\n candidate_round: None,\n panel: vec![],\n verdict: Verdict::Accepted,\n }\n}\n```",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"base": {
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
"base_inventory_digest": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -21,16 +18,6 @@
|
|||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"candidate": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"candidate_inventory_digest": {
|
||||
"type": [
|
||||
"string",
|
||||
|
|
@ -64,17 +51,17 @@
|
|||
"null"
|
||||
]
|
||||
},
|
||||
"floor": {
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
"k": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"threshold": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
"panel": {
|
||||
"description": "One row per panel language, in panel order.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/PanelReading"
|
||||
}
|
||||
},
|
||||
"verdict": {
|
||||
"$ref": "#/$defs/Verdict"
|
||||
|
|
@ -86,10 +73,8 @@
|
|||
"base_rev",
|
||||
"base_inventory_digest",
|
||||
"edit",
|
||||
"base",
|
||||
"floor",
|
||||
"threshold",
|
||||
"verdict"
|
||||
"verdict",
|
||||
"panel"
|
||||
],
|
||||
"$defs": {
|
||||
"Call": {
|
||||
|
|
@ -113,10 +98,54 @@
|
|||
},
|
||||
"Metric": {
|
||||
"description": "Which metric an edit regressed.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Chrf",
|
||||
"Meaning"
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Chrf",
|
||||
"Meaning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "The composite fell, or a reading was not a finite number.",
|
||||
"type": "string",
|
||||
"const": "Composite"
|
||||
}
|
||||
]
|
||||
},
|
||||
"PanelReading": {
|
||||
"description": "One panel language's row of a cycle: the head's Validation reading, the\ncandidate's (none when the edit never reached a round), the language's\nmeasured floor and the threshold derived from it. The threshold decides only\nin a one-language panel ([`decide`]); with two or more languages it is\nrecorded beside the floor and the accept is the sum rule of\n[`decide_panel`].",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"base": {
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
"candidate": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"floor": {
|
||||
"$ref": "#/$defs/Reading"
|
||||
},
|
||||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"threshold": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"language",
|
||||
"base",
|
||||
"floor",
|
||||
"threshold"
|
||||
]
|
||||
},
|
||||
"Reading": {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "RoundRecord",
|
||||
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n }\n}\n```",
|
||||
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::RoundRecord;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n languages: vec![],\n }\n}\n```",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"admit_rules": {
|
||||
|
|
@ -49,6 +49,13 @@
|
|||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"languages": {
|
||||
"description": "One reading per panel language, in panel order. No pooled mean exists.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/LanguageRound"
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -80,27 +87,6 @@
|
|||
"items": {
|
||||
"$ref": "#/$defs/TrainRow"
|
||||
}
|
||||
},
|
||||
"train_mean": {
|
||||
"$ref": "#/$defs/SplitMean"
|
||||
},
|
||||
"train_scored": {
|
||||
"description": "Train items that lowered and scored; `train_mean` is over all of them\nand counts a refused item as 0.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"validation_mean": {
|
||||
"$ref": "#/$defs/SplitMean"
|
||||
},
|
||||
"validation_refused": {
|
||||
"$ref": "#/$defs/RefusalCounts"
|
||||
},
|
||||
"validation_scored": {
|
||||
"description": "Validation items that lowered and scored; `validation_mean` is over all\nof them and counts a refused item as 0, so refusing a weak item cannot\nraise it.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
@ -114,12 +100,8 @@
|
|||
"chrf_version",
|
||||
"calls",
|
||||
"train",
|
||||
"train_mean",
|
||||
"validation_mean",
|
||||
"validation_refused",
|
||||
"train_scored",
|
||||
"validation_scored",
|
||||
"inventory_digest"
|
||||
"inventory_digest",
|
||||
"languages"
|
||||
],
|
||||
"$defs": {
|
||||
"Call": {
|
||||
|
|
@ -141,6 +123,44 @@
|
|||
"items"
|
||||
]
|
||||
},
|
||||
"LanguageRound": {
|
||||
"description": "One panel language's readings in a round: the means over that language's\nitems only (a refused item counts 0) and how many items lowered and scored.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"train_mean": {
|
||||
"$ref": "#/$defs/SplitMean"
|
||||
},
|
||||
"train_scored": {
|
||||
"description": "Train items that lowered and scored; `train_mean` is over all of them.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
},
|
||||
"validation_mean": {
|
||||
"$ref": "#/$defs/SplitMean"
|
||||
},
|
||||
"validation_refused": {
|
||||
"$ref": "#/$defs/RefusalCounts"
|
||||
},
|
||||
"validation_scored": {
|
||||
"description": "Validation items that lowered and scored; `validation_mean` is over all\nof them and counts a refused item as 0, so refusing a weak item cannot\nraise it.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"language",
|
||||
"train_mean",
|
||||
"train_scored",
|
||||
"validation_mean",
|
||||
"validation_scored",
|
||||
"validation_refused"
|
||||
]
|
||||
},
|
||||
"Outcome": {
|
||||
"oneOf": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -49,6 +49,13 @@
|
|||
"description": "The answering model.",
|
||||
"type": "string"
|
||||
},
|
||||
"panel": {
|
||||
"description": "The languages the run offers, reads and decides on: BCP 47-form tags,\nsorted, unique, at least one, set once at open. An item of any other\nlanguage is never offered, minted from, probed or scored.",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"prompt_digest": {
|
||||
"type": "string"
|
||||
}
|
||||
|
|
@ -65,7 +72,8 @@
|
|||
"embedder",
|
||||
"chrf_version",
|
||||
"inventory_digest",
|
||||
"lowerer_reach"
|
||||
"lowerer_reach",
|
||||
"panel"
|
||||
],
|
||||
"$defs": {
|
||||
"LowererReach": {
|
||||
|
|
|
|||
Loading…
Reference in a new issue