diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 28c43db..e225b84 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -8,7 +8,9 @@ //! consts in this module; every prompt also lists the inventory at the run's //! revision. A mint prompt is the editor's: it reads [`TrainItems`], an //! [`Inventory`] and an [`EditFeedback`] and nothing else, so no held-out text, -//! key or per-item score can reach it. +//! key or per-item score can reach it. A lowering prompt names the language it +//! is said in on a language line; every language is named by its tag, none by a +//! default. //! //! Item text may hold line breaks; the renderer indents every continuation line //! (`indented`), so item text cannot forge a `[key]` line or a paragraph. @@ -25,7 +27,7 @@ use serde::Serialize; use crate::corpus::{CorpusItem, CorpusItemId}; use crate::inventory::{Inventory, InventoryEntry, OptionLabel, ValueKind}; -use crate::lower::{EntityForms, LowerError, TARGET_LANGUAGE, entity}; +use crate::lower::{EntityForms, LowerError, entity}; use crate::packet::{ Context, Discourse, EntityId, Extension, Predication, Referent, Relation, Stance, }; @@ -37,7 +39,7 @@ use crate::validate::LoweringInput; /// The instructions a mint prompt starts with: the editor's whole brief. Fixed /// per run by `PROMPT_SOURCE_DIGEST`. pub const MINT_INSTRUCTIONS: &str = "\ -Propose one edit to the inventory below, so that the lines below can be encoded and lowered to English more faithfully. +Propose one edit to the inventory below, so that the lines below can be encoded and lowered, each in its own language, more faithfully. Answer with one JSON object and nothing else: no code fence, no commentary. Shape: {\"add\": [{\"family\", \"gloss\", \"value_kind\", \"applies_to\"}], \"retire\": [ids]}; 1 to 8 changes in all, adds and retires together. - family: one of Concept, Frame, Role, Relation, SpeechAct, Feature, ReferentKind, Extension. @@ -65,7 +67,7 @@ A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": lab /// The instructions a lowering prompt starts with: the lowerer's whole brief. /// Fixed per run by `PROMPT_SOURCE_DIGEST`. pub const LOWER_INSTRUCTIONS: &str = "\ -Say the packet below in English (en-US), as one utterance of its speaker: all of its meaning and nothing it does not carry. +Say the packet below in the language on the language line, as one utterance of its speaker: all of its meaning and nothing it does not carry. Every id is the inventory's listed below and every feature value is as listed; a grade is permille, 0 to 1000. The stance is the speaker's felt affect and restraint: it shapes how the utterance is said, not what it says. A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; @@ -81,14 +83,14 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// reads; it cannot be built from a slice, and no accessor returns text or an /// item. /// -/// ``` +/// ```no_run /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); +/// # let panel = ["en-US".to_string()]; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap(); /// let t = run.train().unwrap(); -/// assert!(t.is_empty()); +/// let _ = t.len(); /// ``` /// /// ```compile_fail @@ -100,7 +102,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let _u = t.clone(); /// ``` @@ -111,7 +113,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// s(&t); /// ``` @@ -120,11 +122,14 @@ pub struct TrainItems { } impl TrainItems { - pub(crate) fn from_corpus(corpus: &CorpusStore) -> anyhow::Result { + /// The live Train items whose language is in `panel`. + pub(crate) fn from_corpus(corpus: &CorpusStore, panel: &[String]) -> anyhow::Result { let items = corpus .all()? .into_iter() - .filter(|item| SplitScheme::V1.live(item) == Some(Split::Train)) + .filter(|item| { + SplitScheme::V1.live(item) == Some(Split::Train) && panel.contains(&item.language) + }) .collect(); Ok(Self { items }) } @@ -252,11 +257,11 @@ impl<'cp> EncoderOffer<'cp> { } /// What the editor is told of how its last edit fared: whether it was kept, and -/// the held-out composite at the current inventory, rounded to the nearest -/// [`LADDER_STEP`]. The head composite changes only when an edit is accepted and -/// an accept needs a gain above one step, so a run releases at most -/// `1 / LADDER_STEP` distinct values (Blum and Hardt 2015, the Ladder). Nothing -/// else about the held-out split is a field. +/// the held-out composite at the current inventory of each panel language, +/// rounded to the nearest [`LADDER_STEP`]. A head composite changes only when an +/// edit is accepted and an accept needs a gain above one step, so a run releases +/// at most `1 / LADDER_STEP` distinct values per language (Blum and Hardt 2015, +/// the Ladder). Nothing else about the held-out split is a field. /// /// ``` /// let f = weksa::context::EditFeedback::none(); @@ -266,18 +271,18 @@ impl<'cp> EncoderOffer<'cp> { /// Its fields are private: /// /// ```compile_fail,E0451 -/// let _ = weksa::context::EditFeedback { last: None, head: None }; +/// let _ = weksa::context::EditFeedback { last: None, head: vec![] }; /// ``` /// /// and only the run builds one with a reading: /// /// ```compile_fail,E0624 -/// let _ = weksa::context::EditFeedback::new(Some(true), Some(0.5)); +/// let _ = weksa::context::EditFeedback::new(Some(true), vec![]); /// ``` -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct EditFeedback { last: Option, - head: Option, + head: Vec<(String, f64)>, } impl EditFeedback { @@ -285,16 +290,20 @@ impl EditFeedback { pub fn none() -> Self { Self { last: None, - head: None, + head: Vec::new(), } } - /// `last`: whether the previous edit was accepted; `head`: the composite at - /// the current inventory, rounded here to the nearest [`LADDER_STEP`]. - pub(crate) fn new(last: Option, head: Option) -> Self { + /// `last`: whether the previous edit was accepted; `head`: each panel + /// language's composite at the current inventory, in panel order, rounded + /// here to the nearest [`LADDER_STEP`]. + pub(crate) fn new(last: Option, head: Vec<(String, f64)>) -> Self { Self { last, - head: head.map(|x| (x / LADDER_STEP).round() * LADDER_STEP), + head: head + .into_iter() + .map(|(tag, x)| (tag, (x / LADDER_STEP).round() * LADDER_STEP)) + .collect(), } } @@ -304,7 +313,15 @@ impl EditFeedback { Some(false) => "rejected", None => "none", }; - let head = self.head.map_or("none".to_string(), |x| format!("{x:.3}")); + let head = if self.head.is_empty() { + "none".to_string() + } else { + self.head + .iter() + .map(|(tag, x)| format!("{tag} {x:.3}")) + .collect::>() + .join(", ") + }; format!("last edit: {last}; validation composite at head: {head}") } } @@ -312,12 +329,12 @@ impl EditFeedback { /// Text for a model call, and the keys of the items it holds. The only /// constructors are [`Prompt::mint`], [`Prompt::encode`] and [`Prompt::lower`]. /// -/// ``` +/// ```no_run /// # let dir = tempfile::tempdir().unwrap(); /// # let p = |n: &str| dir.path().join(n); -/// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); +/// # let panel = ["en-US".to_string()]; +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &panel).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -335,7 +352,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -348,7 +365,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, &["en-US".to_string()]).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -502,17 +519,19 @@ impl Prompt { } } - /// [`LOWER_INSTRUCTIONS`], the meaning of `input` (no kind, constraints, - /// scene, speaker or group), the en-US forms of every entity its referents - /// cite, and the entries of `inventory` live at the view's revision. `item` + /// [`LOWER_INSTRUCTIONS`], the `language: ` line, the meaning of + /// `input` (no kind, constraints, scene, speaker or group), the forms in + /// `language` of every entity its referents cite, and the entries of `inventory` live at the view's revision. `item` /// goes only to [`Prompt::item_keys`], never into the text. The view's /// speaker and group fields are never read; a speaker's entity id appears /// only when the encoder cites that entity on a referent, which is naming a /// cited entity like any other. A cited entity the port lacks, or one with - /// no en-US form, is an error and no prompt. + /// no form in `language`, is an error and no prompt. The caller chooses the + /// language; the view has none. pub fn lower( input: &LoweringInput, item: &CorpusItemId, + language: &str, inventory: &Inventory, entities: &dyn EntityForms, ) -> Result { @@ -520,7 +539,7 @@ impl Prompt { for id in input.referents().iter().filter_map(|r| r.entity.as_ref()) { let found = entity(entities, id)?; let named = found - .forms_in(TARGET_LANGUAGE) + .forms_in(language) .filter(|f| !f.is_empty()) .ok_or_else(|| LowerError::NoForm { entity: id.clone() })?; forms.insert(id.as_str(), named.to_vec()); @@ -536,7 +555,7 @@ impl Prompt { }; let rev = input.inventory_rev(); let text = format!( - "{LOWER_INSTRUCTIONS}\n\npacket: {}\n\nentities: {}\n\ninventory rev:{rev}{}", + "{LOWER_INSTRUCTIONS}\n\nlanguage: {language}\n\npacket: {}\n\nentities: {}\n\ninventory rev:{rev}{}", serde_json::to_string(&meaning).expect("a meaning is plain data"), serde_json::to_string(&forms).expect("forms are plain data"), inventory_lines(inventory, rev) @@ -583,34 +602,39 @@ mod tests { #[test] fn the_feedback_line_prints_exact_values() { + let en = |x: f64| vec![("en-US".to_string(), x)]; let line = |last, head| EditFeedback::new(last, head).line(); assert_eq!( EditFeedback::none().line(), "last edit: none; validation composite at head: none" ); assert_eq!( - line(Some(true), Some(0.5)), - "last edit: accepted; validation composite at head: 0.500" + line(Some(true), en(0.5)), + "last edit: accepted; validation composite at head: en-US 0.500" ); assert_eq!( - line(Some(false), Some(0.0)), - "last edit: rejected; validation composite at head: 0.000" + line(Some(false), en(0.0)), + "last edit: rejected; validation composite at head: en-US 0.000" ); - assert_eq!(line(None, None), EditFeedback::none().line()); + assert_eq!(line(None, vec![]), EditFeedback::none().line()); // The head is rounded to the nearest step before it is printed. assert_eq!( - line(None, Some(0.5012)), - "last edit: none; validation composite at head: 0.500" + line(None, en(0.5012)), + "last edit: none; validation composite at head: en-US 0.500" ); assert_eq!( - line(None, Some(0.5026)), - "last edit: none; validation composite at head: 0.505" + line(None, en(0.5026)), + "last edit: none; validation composite at head: en-US 0.505" ); assert_eq!( - line(None, Some(0.9999)), - "last edit: none; validation composite at head: 1.000" + line(None, en(0.9999)), + "last edit: none; validation composite at head: en-US 1.000" + ); + // One value per panel language, in the order given. + assert_eq!( + line(None, vec![("en-US".into(), 0.5026), ("ja-JP".into(), 0.41)]), + "last edit: none; validation composite at head: en-US 0.505, ja-JP 0.410" ); - assert_eq!(EditFeedback::new(None, Some(0.5026)).head, Some(0.505)); } #[test] diff --git a/crates/weksa/src/lower.rs b/crates/weksa/src/lower.rs index e8e1ea9..76f3158 100644 --- a/crates/weksa/src/lower.rs +++ b/crates/weksa/src/lower.rs @@ -43,8 +43,10 @@ use crate::phones::{Phone, PhoneInventory, SegmentError}; use crate::store::EntityStore; use crate::validate::LoweringInput; -/// The language tag this lowerer's entity forms are read under, spelled as an -/// entity's `LanguageForms::language` and a corpus item's language are. +/// The language tag of the template table and the phones: what `weksa lower` +/// says in. The golf loop lowers each item in its own language and does not +/// read this. Spelled as an entity's `LanguageForms::language` and a corpus +/// item's language are. pub const TARGET_LANGUAGE: &str = "en-US"; /// The lowerer's whole view of the entity store: one entity by id. Narrow so a diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index a92c010..8270565 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -99,6 +99,10 @@ enum LoopCommand { /// that family wrote. Defaults to unrecorded. #[arg(long, default_value = "unrecorded")] answerer_family: ModelFamilyArg, + /// The languages the run offers, reads and decides on: BCP 47 tags, + /// comma-separated. Every one needs a live Train and Validation item. + #[arg(long, value_delimiter = ',', required = true)] + panel: Vec, }, /// Run one round of the run at ROOT. The stores are read only from the /// run's record: the global --corpus, --inventory and --entities are not @@ -329,7 +333,8 @@ struct AddArgs { /// NFC-normalized here; `--name` byte offsets index the normalized text. #[arg(long)] text: String, - #[arg(long, default_value = "en-US")] + /// The item's language, a BCP 47 tag; no language is the default. + #[arg(long)] language: String, /// An entity id. #[arg(long)] @@ -765,6 +770,7 @@ fn main() -> Result<()> { embed_model, lowerer_reach, answerer_family, + panel, }) => { let run = RunStores::create( &root, @@ -778,6 +784,7 @@ fn main() -> Result<()> { &embed_model, lowerer_reach, answerer_family.into(), + &panel, )?; println!("opened run:{}", run.record().label); } @@ -821,12 +828,12 @@ fn main() -> Result<()> { std::process::exit(3); } AblationOutcome::Read(readings) => { - for (split, reading) in readings { + for (language, split, reading) in readings { let name = match split { Split::Train => "train", _ => "validation", }; - println!("{name} {}", reading.line()); + println!("{language} {name} {}", reading.line()); } } } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index dbbf6ad..e6bfb1b 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -36,11 +36,12 @@ use cultcache_rs::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use unicode_segmentation::UnicodeSegmentation; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; -use crate::form::{is_attestation, is_slug}; +use crate::form::{is_attestation, is_language_tag, is_slug}; use crate::model::{ModelPort, PortError}; use crate::packet::{Packet, Stance}; use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; @@ -52,8 +53,8 @@ use crate::validate::LoweringInput; /// The digest of the prompt builder's source: `context.rs`, then `split.rs`, /// whose scheme and digest function decide which lines the prompt holds as -/// Train and name every prompt, then `lower.rs`, whose `TARGET_LANGUAGE` and -/// `EntityForms` port the lowering prompt reads, then `packet.rs`, whose serde +/// Train and name every prompt, then `lower.rs`, whose `EntityForms` port and +/// `entity()` the lowering prompt reads, then `packet.rs`, whose serde /// shapes the lowering prompt prints as the packet. Each file enters the digest /// behind its length, so the parts cannot be re-cut into other parts with the /// same digest. Any edit to any of them, comments included, ends a run; code @@ -117,6 +118,11 @@ pub struct RunRecord { #[cultcache(key = 13, default)] #[schemars(default)] pub answerer_family: ModelFamily, + /// The languages the run offers, reads and decides on: BCP 47-form tags, + /// sorted, unique, at least one, set once at open. An item of any other + /// language is never offered, minted from, probed or scored. + #[cultcache(key = 14)] + pub panel: Vec, } /// How a run's lowering prompts reach the model that answers them (rulings @@ -200,6 +206,22 @@ pub struct RefusalCounts { pub lower: u32, } +/// One panel language's readings in a round: the means over that language's +/// items only (a refused item counts 0) and how many items lowered and scored. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LanguageRound { + pub language: String, + pub train_mean: SplitMean, + /// Train items that lowered and scored; `train_mean` is over all of them. + pub train_scored: u32, + pub validation_mean: SplitMean, + /// Validation items that lowered and scored; `validation_mean` is over all + /// of them and counts a refused item as 0, so refusing a weak item cannot + /// raise it. + pub validation_scored: u32, + pub validation_refused: RefusalCounts, +} + /// One scored round. It holds no text and no Test reading: per-item rows for /// Train only, Validation only as means and refusal counts by stage. Only /// `run::round` builds one. @@ -212,8 +234,7 @@ pub struct RefusalCounts { /// is the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`. /// /// ```compile_fail,E0451 -/// use weksa::run::{RefusalCounts, RoundRecord}; -/// use weksa::score::SplitMean; +/// use weksa::run::RoundRecord; /// fn f() -> RoundRecord { /// RoundRecord { /// n: 1, @@ -226,16 +247,12 @@ pub struct RefusalCounts { /// chrf_version: String::new(), /// calls: vec![], /// train: vec![], -/// train_mean: SplitMean::default(), -/// train_scored: 0, -/// validation_mean: SplitMean::default(), -/// validation_scored: 0, -/// validation_refused: RefusalCounts::default(), /// inventory_digest: String::new(), /// cycle: None, /// admit_rules: vec![], /// parse_reasons: vec![], /// own_family: 0, +/// languages: vec![], /// } /// } /// ``` @@ -265,21 +282,8 @@ pub struct RoundRecord { calls: Vec, #[cultcache(key = 10)] train: Vec, - #[cultcache(key = 11)] - train_mean: SplitMean, - #[cultcache(key = 12)] - validation_mean: SplitMean, - #[cultcache(key = 13)] - validation_refused: RefusalCounts, - /// Train items that lowered and scored; `train_mean` is over all of them - /// and counts a refused item as 0. - #[cultcache(key = 14)] - train_scored: u32, - /// Validation items that lowered and scored; `validation_mean` is over all - /// of them and counts a refused item as 0, so refusing a weak item cannot - /// raise it. - #[cultcache(key = 15)] - validation_scored: u32, + // Keys 11 to 15 (the pooled means and counts) are retired and never reused: + // a run.cc from before does not open. /// 16 hex of the inventory file the round was scored under. #[cultcache(key = 16)] inventory_digest: String, @@ -304,6 +308,10 @@ pub struct RoundRecord { #[cultcache(key = 20, default)] #[schemars(default)] own_family: u32, + /// One reading per panel language, in panel order. No pooled mean exists. + // Key 20 is `own_family`; the per-language readings take 21. + #[cultcache(key = 21)] + languages: Vec, } impl RoundRecord { @@ -337,20 +345,12 @@ impl RoundRecord { pub fn train(&self) -> &[TrainRow] { &self.train } - pub fn train_mean(&self) -> &SplitMean { - &self.train_mean + /// The panel languages' readings, in panel order. + pub fn languages(&self) -> &[LanguageRound] { + &self.languages } - pub fn validation_mean(&self) -> &SplitMean { - &self.validation_mean - } - pub fn validation_refused(&self) -> RefusalCounts { - self.validation_refused - } - pub fn train_scored(&self) -> u32 { - self.train_scored - } - pub fn validation_scored(&self) -> u32 { - self.validation_scored + pub fn language(&self, tag: &str) -> Option<&LanguageRound> { + self.languages.iter().find(|l| l.language == tag) } pub fn inventory_digest(&self) -> &str { &self.inventory_digest @@ -368,37 +368,38 @@ impl RoundRecord { self.own_family } - /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, - /// `scored` those that lowered; a mean counts a refused item as 0. + /// The round line, unlabelled: [`RunStores::round_line`] adds how the run + /// was lowered. One segment per panel language; `n` is every item of the + /// split, `scored` those that lowered; a mean counts a refused item as 0. fn line(&self) -> String { let mean = |value: Option, places: usize| { value.map_or("none".to_string(), |v| format!("{v:.places$}")) }; - let refused = self - .train - .iter() - .filter(|row| matches!(row.outcome, Outcome::Refused(_))) - .count(); - let (t, v, x) = ( - &self.train_mean, - &self.validation_mean, - self.validation_refused, - ); + let segment = |l: &LanguageRound| { + let (t, v, x) = (&l.train_mean, &l.validation_mean, l.validation_refused); + let refused = t.count.saturating_sub(l.train_scored as usize); + format!( + "{} train n={} scored={} chrf={} meaning={} refused={refused} validation n={} scored={} chrf={} meaning={} refused parse={} admit={} lower={}", + l.language, + t.count, + l.train_scored, + mean(t.chrf, 2), + mean(t.meaning, 4), + v.count, + l.validation_scored, + mean(v.chrf, 2), + mean(v.meaning, 4), + x.parse, + x.admit, + x.lower, + ) + }; + let segments: Vec = self.languages.iter().map(segment).collect(); let line = format!( - "round {} rev {} train n={} scored={} chrf={} meaning={} refused={refused} validation n={} scored={} chrf={} meaning={} refused parse={} admit={} lower={}", + "round {} rev {} {}", self.n, self.inventory_rev, - t.count, - self.train_scored, - mean(t.chrf, 2), - mean(t.meaning, 4), - v.count, - self.validation_scored, - mean(v.chrf, 2), - mean(v.meaning, 4), - x.parse, - x.admit, - x.lower, + segments.join(" ") ); let codes = |counts: &[RuleCount]| { counts @@ -458,6 +459,9 @@ pub enum RunError { /// The round holds no answer to a lowering prompt the ablation rebuilt: the /// prompt's digest drifted or the head inventory moved since the round. Unrecorded, + /// The panel is empty, repeats or misspells a language, or names one with no + /// live Train or Validation item in the pinned corpus. + Panel, } impl fmt::Display for RunError { @@ -479,6 +483,9 @@ impl fmt::Display for RunError { RunError::Embed => f.write_str("the embedder failed"), RunError::NoFloor => f.write_str("the run has no noise floor: record two rounds first"), RunError::NoValidation => f.write_str("a reading has no validation item"), + RunError::Panel => f.write_str( + "the panel is empty, repeats or misspells a language, or names one with no live Train or Validation item", + ), } } } @@ -586,8 +593,11 @@ fn copy_new(from: &Path, to: &Path) -> Result<(), RunError> { } impl RunStores { - /// Pins the three stores at `root`, which must hold no run. A store file - /// that does not exist is `Io(NotFound)`. + /// Pins the three stores at `root`, which must hold no run, and the panel: + /// the languages the run offers, reads and decides on. A store file that + /// does not exist is `Io(NotFound)`; a bad panel is `Panel` and writes + /// nothing. + #[allow(clippy::too_many_arguments)] pub fn create( root: &Path, label: &str, @@ -596,6 +606,7 @@ impl RunStores { embedder: &str, reach: LowererReach, answerer_family: ModelFamily, + panel: &[String], ) -> Result { Self::create_between( root, @@ -605,6 +616,7 @@ impl RunStores { embedder, reach, answerer_family, + panel, || {}, ) } @@ -612,7 +624,7 @@ impl RunStores { /// `create` with `between` run after the digests are taken and before the /// stores are opened: the seam that lets a test change a store in that window. #[doc(hidden)] - #[allow(clippy::too_many_arguments)] // `create`'s six plus the test seam + #[allow(clippy::too_many_arguments)] // `create`'s seven plus the test seam pub fn create_between( root: &Path, label: &str, @@ -621,6 +633,7 @@ impl RunStores { embedder: &str, reach: LowererReach, answerer_family: ModelFamily, + panel: &[String], between: impl FnOnce(), ) -> Result { if !is_slug(label) { @@ -632,6 +645,14 @@ impl RunStores { if !is_attestation(embedder) { return Err(RunError::Malformed("embedder")); } + let mut panel = panel.to_vec(); + panel.sort(); + if panel.is_empty() + || !panel.iter().all(|tag| is_language_tag(tag)) + || panel.windows(2).any(|w| w[0] == w[1]) + { + return Err(RunError::Panel); + } let cache = run_cache(root)?; if one_run(&cache)?.is_some() { return Err(RunError::Exists); @@ -650,6 +671,7 @@ impl RunStores { inventory_digest: String::new(), lowerer_reach: reach, answerer_family, + panel, }; // The run works on its own copy; the caller's store is never opened again. let head = root.join(INVENTORY_FILE); @@ -665,6 +687,7 @@ impl RunStores { }; between(); let mut run = Self::with(root, cache, record)?; + run.check_panel()?; // The stores were read between the digests: unless they still match // the pins, what was opened is not what was pinned. Nothing is // recorded for a run that did not pin. @@ -724,6 +747,26 @@ impl RunStores { &self.record } + /// Every panel language holds a live Train and a live Validation item. + fn check_panel(&self) -> Result<(), RunError> { + let items = self.corpus.all().map_err(|_| RunError::Store)?; + let holds = |tag: &String, split: Split| { + items + .iter() + .any(|i| &i.language == tag && SplitScheme::V1.live(i) == Some(split)) + }; + if self + .record + .panel + .iter() + .all(|tag| holds(tag, Split::Train) && holds(tag, Split::Validation)) + { + Ok(()) + } else { + Err(RunError::Panel) + } + } + /// `line` (a round or cycle summary) with how this run's lowerings were /// answered; a briefed-subagent run says its numbers are smoke-round only. pub fn labelled(&self, line: &str) -> String { @@ -755,9 +798,9 @@ impl RunStores { &self.entities } - /// The live Train items of the pinned corpus. + /// The live Train items of the pinned corpus in a panel language. pub fn train(&self) -> anyhow::Result { - TrainItems::from_corpus(&self.corpus) + TrainItems::from_corpus(&self.corpus, &self.record.panel) } /// The offer for `key`; the inner error is the gate's refusal. An absent @@ -769,17 +812,20 @@ impl RunStores { let item = self .corpus .get(key)? + .filter(|item| self.record.panel.contains(&item.language)) .ok_or_else(|| anyhow::anyhow!("no such item"))?; Ok(EncoderOffer::for_round(&item)) } - /// The live Train and Validation offers, in key order. Items the gate - /// refuses are skipped and read no further. + /// The live Train and Validation offers in a panel language, in key order. + /// Items the gate refuses, and items of any other language, are skipped + /// and read no further. pub fn offers(&self) -> anyhow::Result>> { Ok(self .corpus .all()? .iter() + .filter(|item| self.record.panel.contains(&item.language)) .filter_map(|item| EncoderOffer::for_round(item).ok()) .collect()) } @@ -862,11 +908,19 @@ pub enum RoundOutcome { /// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model /// completes each live Train and Validation item from its opening alone, chrF against the -/// item's own second half. Recorded nowhere: the answers stay in `/recall`. -/// It is never a gate; a high reading says a round's scores on these items are -/// recitation. +/// item's own second half, one reading per panel language. Recorded nowhere: +/// the answers stay in `/recall`. It is never a gate; a high reading says +/// a round's scores on these items are recitation. #[derive(Clone, Debug, PartialEq)] pub struct RecallReading { + /// In panel order. + pub languages: Vec, +} + +/// One panel language's recall reading. +#[derive(Clone, Debug, PartialEq)] +pub struct LanguageRecall { + pub language: String, /// Items probed, Train and Validation. pub n: u32, /// Items whose completion was a usable answer; a refused one counts 0. @@ -876,10 +930,24 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { - format!( - "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 14 to 20, so a reading near that floor is no recitation and one well above it is)", - self.n, self.answered, self.chrf - ) + self.languages + .iter() + .map(|l| { + let segment = format!( + "recall {} n={} answered={} chrf={:.2}", + l.language, l.n, l.answered, l.chrf + ); + // The one language whose unrelated-text floor was measured. + if l.language == "en-US" { + format!( + "{segment} (completion from the opening; unrelated English already scores chrf 14 to 20, so a reading near that floor is no recitation and one well above it is)" + ) + } else { + segment + } + }) + .collect::>() + .join(" | ") } } @@ -889,20 +957,27 @@ pub enum RecallOutcome { Read(RecallReading), } -/// The opening and the rest of `text`, split between words at about the -/// middle; `None` when it has fewer than four words. +/// The opening and the rest of `text`, split at the Unicode word boundary +/// (UAX #29) nearest the byte middle; `None` when it has fewer than four words. +/// One rule for every script: a line written without spaces is probed like any +/// other. fn recall_split(text: &str) -> Option<(String, String)> { - let words: Vec<&str> = text.split_whitespace().collect(); - if words.len() < 4 { + if text.unicode_words().count() < 4 { return None; } - let (head, tail) = words.split_at(words.len() / 2); - Some((head.join(" "), tail.join(" "))) + let middle = text.len() / 2; + let at = text + .split_word_bound_indices() + .map(|(i, _)| i) + .filter(|&i| i > 0) + .min_by_key(|&i| i.abs_diff(middle))?; + let (head, tail) = text.split_at(at); + Some((head.trim().to_string(), tail.trim().to_string())) } impl RunStores { - /// Probes every live Train and Validation item of the pinned corpus once, - /// through `model`. A public-domain item names its work, author and + /// Probes every live Train and Validation item of the pinned corpus in a + /// panel language once, through `model`. A public-domain item names its work, author and /// translator; any other says `work: not published`. pub fn recall(&self, model: &mut dyn ModelPort) -> Result { let store = |_| RunError::Store; @@ -917,41 +992,60 @@ impl RunStores { } => format!("work: {work}\nauthor: {author}\ntranslator: {translator}"), _ => "work: not published".to_string(), }; - if EncoderOffer::for_round(&item).is_err() { + if !self.record.panel.contains(&item.language) + || EncoderOffer::for_round(&item).is_err() + { continue; } if let Some((opening, rest)) = recall_split(&item.text) { let prompt = Prompt::recall(&item.id, &provenance, &opening); - probes.push((prompt, rest)); + probes.push((item.language.clone(), prompt, rest)); } } let mut unanswered = Vec::new(); let mut scores = Vec::new(); - for (prompt, rest) in &probes { + for (language, prompt, rest) in &probes { match model.answer(prompt) { Ok(answer) => { let text = LoweringDraft::parse(&answer) .ok() .filter(LoweringDraft::usable) .map(|d| d.text); - scores.push(text.map(|t| chrf(&t, rest))); + scores.push((language, text.map(|t| chrf(&t, rest)))); } Err(PortError::Unanswered) => unanswered.push(prompt.digest()), - Err(PortError::Io(ErrorKind::InvalidData)) => scores.push(None), + Err(PortError::Io(ErrorKind::InvalidData)) => scores.push((language, None)), Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), } } if !unanswered.is_empty() { return Ok(RecallOutcome::Unanswered { prompts: unanswered }); } - let n = scores.len() as u32; - let answered = scores.iter().flatten().count() as u32; - let chrf = if n == 0 { - 0.0 - } else { - scores.iter().map(|s| s.unwrap_or(0.0)).sum::() / f64::from(n) - }; - Ok(RecallOutcome::Read(RecallReading { n, answered, chrf })) + let languages = self + .record + .panel + .iter() + .map(|tag| { + let of: Vec> = scores + .iter() + .filter(|(language, _)| *language == tag) + .map(|(_, score)| *score) + .collect(); + let n = of.len() as u32; + let chrf = if n == 0 { + 0.0 + } else { + of.iter().map(|s| s.unwrap_or(0.0)).sum::() / f64::from(n) + }; + LanguageRecall { + language: tag.clone(), + n, + answered: of.iter().flatten().count() as u32, + chrf, + } + }) + .collect(); + Ok(RecallOutcome::Read(RecallReading { languages })) } /// `/recall`, where a recall's prompts and answers live. @@ -1028,8 +1122,8 @@ impl AblationReading { #[derive(Debug)] pub enum AblationOutcome { Unanswered { prompts: Vec }, - /// Train then Validation. - Read(Vec<(Split, AblationReading)>), + /// Per panel language in panel order, Train then Validation. + Read(Vec<(String, Split, AblationReading)>), } /// What the port made of one lowering prompt. @@ -1070,7 +1164,7 @@ impl RunStores { } /// A diagnostic like [`recall`](Self::recall), never a gate: for each live - /// Train and Validation packet of plain round `n`, asks the lowerer again + /// Train and Validation packet of plain round `n`, per panel language, asks the lowerer again /// (`model`, the replay), with the packet's stance emptied (drop) and with /// the next item's stance in its split (permute), and reads how far the /// lowering moves against the round's own answer (`recorded`). Test and @@ -1111,16 +1205,20 @@ impl RunStores { .map_err(store)? .all() .map_err(store)?; - let mut splits: [(Split, Vec<(CorpusItem, Packet)>); 2] = - [(Split::Train, Vec::new()), (Split::Validation, Vec::new())]; + let mut splits: Vec<(String, Split, Vec<(CorpusItem, Packet)>)> = Vec::new(); + for tag in &self.record.panel { + for split in [Split::Train, Split::Validation] { + splits.push((tag.clone(), split, Vec::new())); + } + } for packet in round_packets { let item = self .corpus .get(packet.provenance.corpus_item.as_str()) .map_err(store)? .ok_or(RunError::Store)?; - for (split, held) in &mut splits { - if SplitScheme::V1.live(&item) == Some(*split) { + for (language, split, held) in &mut splits { + if *language == item.language && SplitScheme::V1.live(&item) == Some(*split) { held.push((item.clone(), packet.clone())); } } @@ -1130,8 +1228,8 @@ impl RunStores { let mut readings = Vec::new(); // Texts per split: (item text, recorded, replay, [drop, permute]). let mut texts: Vec> = Vec::new(); - let mut unusable = [0u32; 2]; - for (at, (_, held)) in splits.iter_mut().enumerate() { + let mut unusable = vec![0u32; splits.len()]; + for (at, (_, _, held)) in splits.iter_mut().enumerate() { held.sort_by(|a, b| a.0.id.as_str().cmp(b.0.id.as_str())); let mut rows = Vec::new(); for (i, (item, packet)) in held.iter().enumerate() { @@ -1140,7 +1238,7 @@ impl RunStores { let view = stores[slot] .admit(packet, &self.inventory, &self.corpus, &self.entities) .map_err(|_| RunError::Store)?; - Prompt::lower(&view, &packet_item, &inventory, &self.entities) + Prompt::lower(&view, &packet_item, &item.language, &inventory, &self.entities) .map_err(|_| RunError::Store) }; let base = prompt_of(0, packet.clone())?; @@ -1215,7 +1313,7 @@ impl RunStores { let mut pair = || -> Result { Ok((scores.next().ok_or(RunError::Store)?, scores.next().ok_or(RunError::Store)?)) }; - for (at, ((split, _), rows)) in splits.iter().zip(&texts).enumerate() { + for (at, ((language, split, _), rows)) in splits.iter().zip(&texts).enumerate() { let mut replay: Vec = Vec::new(); let mut variants: [Vec<(Scored, Scored)>; 2] = [Vec::new(), Vec::new()]; for (_, _, _, got) in rows { @@ -1238,6 +1336,7 @@ impl RunStores { ), }; readings.push(( + language.clone(), *split, AblationReading { n: rows.len() as u32, @@ -1418,7 +1517,10 @@ fn score_round( // port. Nothing is kept unless every one is answered. let mut unanswered = Vec::new(); for (at, id, packet_key, view) in &admitted { - let Ok(prompt) = Prompt::lower(view, id, &inventory, &run.entities) else { + // Each item is lowered in its own language: the only reference the + // scorer has is the item's own text. + let language = &items[*at].language; + let Ok(prompt) = Prompt::lower(view, id, language, &inventory, &run.entities) else { fates[*at] = Fate::Refused(Stage::Lower, Some(packet_key.clone())); continue; }; @@ -1497,11 +1599,25 @@ fn score_round( .zip(&score_at) .map(|(item, score)| (item, score.clone().unwrap_or_else(|| nothing.clone()))) .collect(); - let means = aggregate(&scored); let mut train_rows = Vec::new(); - let mut refused = RefusalCounts::default(); - let (mut train_scored, mut validation_scored) = (0, 0); + let mut languages: Vec = Vec::new(); + for tag in &run.record.panel { + let of: Vec<(&CorpusItem, ItemScore)> = scored + .iter() + .filter(|(item, _)| &item.language == tag) + .cloned() + .collect(); + let means = aggregate(&of); + languages.push(LanguageRound { + language: tag.clone(), + train_mean: means.train, + train_scored: 0, + validation_mean: means.validation, + validation_scored: 0, + validation_refused: RefusalCounts::default(), + }); + } for ((item, fate), score) in items.iter().zip(&fates).zip(&score_at) { let (outcome, packet) = match (fate, score) { (Fate::Lowered(_, key), Some(score)) => ( @@ -1514,20 +1630,24 @@ fn score_round( (Fate::Refused(stage, key), _) => (Outcome::Refused(*stage), key.clone()), (Fate::Lowered(..), None) => return Err(RunError::Store), }; + let round = languages + .iter_mut() + .find(|l| l.language == item.language) + .ok_or(RunError::Store)?; match (SplitScheme::V1.live(item), fate) { (Some(Split::Train), fate) => { - train_scored += u32::from(matches!(fate, Fate::Lowered(..))); + round.train_scored += u32::from(matches!(fate, Fate::Lowered(..))); train_rows.push(TrainRow { item: item.id.clone(), packet, outcome, }); } - (Some(Split::Validation), Fate::Lowered(..)) => validation_scored += 1, + (Some(Split::Validation), Fate::Lowered(..)) => round.validation_scored += 1, (Some(Split::Validation), Fate::Refused(stage, _)) => match stage { - Stage::Parse => refused.parse += 1, - Stage::Admit => refused.admit += 1, - Stage::Lower => refused.lower += 1, + Stage::Parse => round.validation_refused.parse += 1, + Stage::Admit => round.validation_refused.admit += 1, + Stage::Lower => round.validation_refused.lower += 1, }, _ => {} } @@ -1554,11 +1674,6 @@ fn score_round( chrf_version: run.record.chrf_version.clone(), calls, train: train_rows, - train_mean: means.train, - validation_mean: means.validation, - validation_refused: refused, - train_scored, - validation_scored, inventory_digest: inventory_digest.to_string(), cycle, admit_rules: admit_rules @@ -1570,6 +1685,7 @@ fn score_round( .map(|(rule, count)| RuleCount { rule, count }) .collect(), own_family, + languages, }; Ok(RoundOutcome::Recorded(record)) } @@ -1654,6 +1770,44 @@ pub fn decide(base: &Reading, candidate: &Reading, floor: &Reading) -> Verdict { } } +/// The accept rule across the panel, over one `(base, candidate, floor)` row +/// per panel language in panel order: [`decide`] per language, unchanged, then +/// Pareto non-regression. Any language that regresses rejects (the first in +/// panel order names the metric); otherwise the edit is accepted when at least +/// one language gains past its own threshold, and rejected as no gain when none +/// does. A language never buys another's regression: Wang, Lipton and Tsvetkov +/// (EMNLP 2020) measure negative interference between languages sharing +/// parameters, and the convergence test reads the same way (no accepted change +/// raised any panel language's composite). +pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict { + let verdicts: Vec = rows + .iter() + .map(|(base, candidate, floor)| decide(base, candidate, floor)) + .collect(); + if let Some(regressed) = verdicts + .iter() + .find(|v| matches!(v, Verdict::Rejected(Why::Regressed(_)))) + { + *regressed + } else if verdicts.contains(&Verdict::Accepted) { + Verdict::Accepted + } else { + Verdict::Rejected(Why::NoGain) + } +} + +/// One panel language's row of a cycle: the head's Validation reading, the +/// candidate's (none when the edit never reached a round), the language's +/// measured floor and the threshold derived from it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PanelReading { + pub language: String, + pub base: Reading, + pub candidate: Option, + pub floor: Reading, + pub threshold: f64, +} + /// One closed edit-accept cycle: digests and numbers, no text. Key /// `run: