GameCult/Weksa
Watch
9
0
Fork
You've already forked Weksa
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

round: store per-item Validation scores, numbers only and private to run.rs, for the paired cycle floor (key 22, defaulted so old runs open)

Browse source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Weksa 2026-11-13 03:31:10 +00:00
commit ddaa9be61b
5 changed files with 264 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

25
crates/weksa/src/run.rs
View file

@ -46,7 +46,7 @@ use crate::encode::{
use crate::form::{is_attestation, is_language_tag, is_slug};
use crate::model::{AnswerFiles, ModelPort, PortError};
use crate::packet::{Packet, Stance};
use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate};
use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items};
use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
use crate::store::{
CorpusStore, EntityStore, InventoryStore, OntologyStore, PacketStore, SpeakerStateStore,
@ -246,6 +246,14 @@ pub struct LanguageRound {
pub validation_refused: RefusalCounts,
}
/// One panel language's Validation scores in a round, in corpus order: numbers
/// only, no item key and no text. Only `run.rs` reads them (the paired floor).
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct LanguageScores {
pub language: String,
pub scores: Vec<ItemScore>,
}
/// One scored round. It holds no text and no Test reading: per-item rows for
/// Train only, Validation only as means and refusal counts by stage. Only
/// `run::round` builds one.
@ -277,6 +285,7 @@ pub struct LanguageRound {
/// parse_reasons: vec![],
/// own_family: 0,
/// languages: vec![],
/// validation_scores: vec![],
/// }
/// }
/// ```
@ -336,6 +345,12 @@ pub struct RoundRecord {
// Key 20 is `own_family`; the per-language readings take 21.
#[cultcache(key = 21)]
languages: Vec<LanguageRound>,
/// Each panel language's live Validation scores in corpus order, a refused
/// item as 0. Private with no getter: only `run.rs` (the floor) reads them,
/// so a leak outside it is a compile error. A run from before has none.
#[cultcache(key = 22, default)]
#[schemars(default)]
validation_scores: Vec<LanguageScores>,
}
impl RoundRecord {
@ -2069,6 +2084,7 @@ fn score_round<M: ModelPort, E: Embedder>(
let mut train_rows = Vec::new();
let mut languages: Vec<LanguageRound> = Vec::new();
let mut validation_scores: Vec<LanguageScores> = Vec::new();
for tag in &run.record.panel {
let of: Vec<(&CorpusItem, ItemScore)> = scored
.iter()
@ -2076,6 +2092,10 @@ fn score_round<M: ModelPort, E: Embedder>(
.cloned()
.collect();
let means = aggregate(&of);
validation_scores.push(LanguageScores {
language: tag.clone(),
scores: validation_items(&of),
});
languages.push(LanguageRound {
language: tag.clone(),
train_mean: means.train,
@ -2153,6 +2173,7 @@ fn score_round<M: ModelPort, E: Embedder>(
.collect(),
own_family,
languages,
validation_scores,
};
Ok(RoundOutcome::Recorded(record))
}
@ -2764,6 +2785,7 @@ mod tests {
parse_reasons: vec![],
own_family: 0,
languages: vec![],
validation_scores: vec![],
};
assert_eq!(r.n(), 1);
}
@ -3276,6 +3298,7 @@ mod tests {
validation_scored: 2,
validation_refused: RefusalCounts::default(),
}],
validation_scores: vec![],
}
}

46
crates/weksa/src/score/mod.rs
View file

@ -28,7 +28,7 @@ pub struct ScoreInput<'a> {
pub output: &'a str,
}
#[derive(Clone, Debug, PartialEq)]
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ItemScore {
/// chrF of the output against the reference.
pub chrf: f64,
@ -131,13 +131,18 @@ pub struct Aggregate {
pub validation: SplitMean,
}
/// The mean of the scores of the items read in split `want`.
fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean {
let of: Vec<&ItemScore> = scored
/// The scores of the items read in split `want`, in list order.
fn scores_in<'a>(scored: &'a [(&CorpusItem, ItemScore)], want: Split) -> Vec<&'a ItemScore> {
scored
.iter()
.filter(|(item, _)| SplitScheme::V1.live(item) == Some(want))
.map(|(_, score)| score)
.collect();
.collect()
}
/// The mean of the scores of the items read in split `want`.
fn split_mean(scored: &[(&CorpusItem, ItemScore)], want: Split) -> SplitMean {
let of = scores_in(scored, want);
let average = |values: Vec<f64>| {
(!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)
};
@ -157,6 +162,12 @@ pub fn aggregate(scored: &[(&CorpusItem, ItemScore)]) -> Aggregate {
}
}
/// The Validation items' scores in list order, numbers only: no key, no text.
/// A refused item arrives scored 0 and keeps its position.
pub fn validation_items(scored: &[(&CorpusItem, ItemScore)]) -> Vec<ItemScore> {
scores_in(scored, Split::Validation).into_iter().cloned().collect()
}
/// The Test split's readings, scalars only: no item key, no minted string, no
/// pair. The checkpoint decision (convergence-test) needs numbers; a key beside
/// a minted string is the handle an edit step would act on. The only producer
@ -249,6 +260,31 @@ mod tests {
item(key, text, &group_in(split), model)
}
#[test]
fn validation_items_keeps_validation_only_in_order() {
let train = in_split("item:t:1", UNRELATED, Split::Train, false);
let first = in_split("item:v:1", "first words", Split::Validation, false);
let refused = in_split("item:v:2", "second words", Split::Validation, false);
let last = in_split("item:v:3", "third words", Split::Validation, false);
let mut gone = in_split("item:v:4", "withdrawn words", Split::Validation, false);
gone.withdrawn = true;
let test = in_split("item:c:1", HELD_OUT, Split::Test, false);
let scored = [
(&train, score(10.0, Some(0.1))),
(&first, score(40.0, Some(0.4))),
(&test, score(60.0, Some(0.6))),
(&refused, score(0.0, Some(0.0))),
(&gone, score(90.0, Some(0.9))),
(&last, score(80.0, None)),
];
let got = validation_items(&scored);
assert_eq!(
got,
[score(40.0, Some(0.4)), score(0.0, Some(0.0)), score(80.0, None)]
);
assert_eq!(got.len(), aggregate(&scored).validation.count);
}
#[test]
fn test_readings_at_checkpoint() {
let train_item = in_split("item:t:1", UNRELATED, Split::Train, false);

149
crates/weksa/tests/golf_loop.rs
View file

@ -2645,6 +2645,9 @@ fn source_tripwires() {
"Aggregate",
"validation_mean",
"validation_refused",
"ItemScore",
"LanguageScores",
"validation_scores",
] {
assert!(!context.contains(banned), "context.rs names {banned}");
}
@ -3364,6 +3367,152 @@ fn a_run_from_before_does_not_open() {
assert!(reopened.rounds().is_err());
}
/// A round as the file holds it, with key 22 the lib keeps private.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_round")]
struct ScoredRound {
#[cultcache(key = 0)]
n: u32,
#[cultcache(key = 1)]
inventory_rev: u32,
#[cultcache(key = 2)]
prompt_digest: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
model: String,
#[cultcache(key = 7)]
embedder: String,
#[cultcache(key = 8)]
chrf_version: String,
#[cultcache(key = 9)]
calls: Vec<weksa::run::Call>,
#[cultcache(key = 10)]
train: Vec<weksa::run::TrainRow>,
#[cultcache(key = 16)]
inventory_digest: String,
#[cultcache(key = 17)]
cycle: Option<u32>,
#[cultcache(key = 18)]
admit_rules: Vec<weksa::run::RuleCount>,
#[cultcache(key = 19)]
parse_reasons: Vec<weksa::run::RuleCount>,
#[cultcache(key = 20)]
own_family: u32,
#[cultcache(key = 21)]
languages: Vec<weksa::run::LanguageRound>,
#[cultcache(key = 22)]
validation_scores: Vec<weksa::run::LanguageScores>,
}
/// A round as written before key 22 existed.
#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)]
#[cultcache(type = "weksa.loop_round")]
struct PreScoresRound {
#[cultcache(key = 0)]
n: u32,
#[cultcache(key = 1)]
inventory_rev: u32,
#[cultcache(key = 2)]
prompt_digest: String,
#[cultcache(key = 4)]
corpus_digest: String,
#[cultcache(key = 5)]
entity_digest: String,
#[cultcache(key = 6)]
model: String,
#[cultcache(key = 7)]
embedder: String,
#[cultcache(key = 8)]
chrf_version: String,
#[cultcache(key = 9)]
calls: Vec<weksa::run::Call>,
#[cultcache(key = 10)]
train: Vec<weksa::run::TrainRow>,
#[cultcache(key = 16)]
inventory_digest: String,
#[cultcache(key = 17)]
cycle: Option<u32>,
#[cultcache(key = 18)]
admit_rules: Vec<weksa::run::RuleCount>,
#[cultcache(key = 19)]
parse_reasons: Vec<weksa::run::RuleCount>,
#[cultcache(key = 20)]
own_family: u32,
#[cultcache(key = 21)]
languages: Vec<weksa::run::LanguageRound>,
}
#[test]
fn a_round_stores_one_score_per_validation_item() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let mean = r.language("en-US").unwrap().validation_mean.clone();
let root = dao.root.clone();
drop(dao.run);
let mut cache = CultCache::new();
cache.register_entry_type::<RunRecord>().unwrap();
cache.register_entry_type::<ScoredRound>().unwrap();
cache
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
.unwrap();
cache.pull_all_backing_stores().unwrap();
let stored = cache.get_all::<ScoredRound>().unwrap().remove(0);
assert_eq!(stored.validation_scores.len(), 1);
let en = &stored.validation_scores[0];
assert_eq!(en.language, "en-US");
assert_eq!(en.scores.len(), mean.count);
let average = en.scores.iter().map(|s| s.chrf).sum::<f64>() / en.scores.len() as f64;
assert!((average - mean.chrf.unwrap()).abs() < 1e-9);
}
#[test]
fn an_old_run_without_scores_still_opens() {
let mut dao = dao();
let mut model = fake(&good_answers());
let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
let rec = dao.run.record().clone();
let root = dao.root.clone();
drop(dao.run);
let old = PreScoresRound {
n: r.n(),
inventory_rev: r.inventory_rev(),
prompt_digest: r.prompt_digest().into(),
corpus_digest: r.corpus_digest().into(),
entity_digest: r.entity_digest().into(),
model: r.model().into(),
embedder: r.embedder().into(),
chrf_version: r.chrf_version().into(),
calls: r.calls().to_vec(),
train: r.train().to_vec(),
inventory_digest: r.inventory_digest().into(),
cycle: r.cycle(),
admit_rules: vec![],
parse_reasons: vec![],
own_family: 0,
languages: vec![r.language("en-US").unwrap().clone()],
};
let _ = std::fs::remove_file(root.join("run.cc"));
{
let mut cache = CultCache::new();
cache.register_entry_type::<RunRecord>().unwrap();
cache.register_entry_type::<PreScoresRound>().unwrap();
cache
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc")))
.unwrap();
cache.pull_all_backing_stores().unwrap();
cache.put("run:dao".to_string(), &rec).unwrap();
cache.put("run:dao:round:1".to_string(), &old).unwrap();
}
let reopened = RunStores::open(&root).unwrap();
let rounds = reopened.rounds().unwrap();
assert_eq!(rounds.len(), 1);
assert_eq!(rounds[0].language("en-US"), r.language("en-US"));
}
#[test]
fn the_published_run_and_round_schemas_require_none_of_the_new_fields() {
let published = weksa::schema::published();

2
docs/targets/interlingua-lowering-map.md
View file

@ -40,7 +40,7 @@ means no store holds it and nothing writes it.
| Scene (cut `speaker-scene`) | Key `scene:<group>@<n>`, the item's split group; type `weksa.scene`: participants (speaker slugs) and, per ordered pair, relation features of slot `Scene` (relative status, intimacy, in-group, audience and whatever the loop adds), at an inventory revision (provenance; a run reads the features at its own head), and an `Annotator` | As Speaker. Keyed by the group, so a scene never straddles the split. `data/original/speaker-state.cc` holds Model-annotated keeper scenes, one per Train or Validation group of both languages (`weksa scene put|dump`); a Test group has none and reads `NoScene` | As Speaker; a model-annotated scene is never Test evidence |
| Corpus item | Key `item:<source-slug>:<n>`, `n` canonical decimal >= 1 (no sign, no leading zero, from cut 2b); type `weksa.corpus_item` with text, language, speaker, split group, prior turns, source. Source `ModelWritten` carries a `ModelFamily` enum, the generator string as the fixture wrote it, and a prompt digest (cut 2b). The item carries no speaker state of its own: its speaker slug and group name the Speaker and Scene documents | Admitted once; the whole item is immutable (re-adding an identical item is a no-op, any difference is refused; a correction is a new item that `supersedes` the old); text already held under another key is refused unless the new item supersedes that key (cut 2b); withdrawn, never deleted, so old scores stay explainable | Importers and `weksa corpus add`, under `corpus-cast-plus-public-domain`; the source enum has no variant for Discord or any training-forbidden source (`corpus-clean`) |
| Split assignment | Derived: `split(scheme, item.group)` | Fixed for a scheme; a new scheme is a new constant, never a reassignment. A model-written item whose group derives test is in no split; a superseding item keeps its predecessor's group | The hash function. Nobody assigns by hand, so there is no writer to forbid |
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, Validation means and refusal counts (from cut `ja-panel-loop`: one set per language of the run's panel, no pooled mean), the digest of the inventory file it was scored under and, for a cycle's candidate round, its cycle `k` | Scored and recorded whole; immutable. A candidate round is put in one batch with its Decision. Lives in the run store on Yggdrasil | `run::round` and `run::cycle` (both through `score_round`) |
| Loop round | Key `run:<label>:round:<n>`; type `weksa.loop_round` with spec version, profile revisions, model ids and prompt digests, sampled train items, Validation means and refusal counts (from cut `ja-panel-loop`: one set per language of the run's panel, no pooled mean), per-item Validation scores in corpus order (key 22, numbers only, no item key or text, private to `run.rs` for the paired floor; a run from before has none and opens), the digest of the inventory file it was scored under and, for a cycle's candidate round, its cycle `k` | Scored and recorded whole; immutable. A candidate round is put in one batch with its Decision. Lives in the run store on Yggdrasil | `run::round` and `run::cycle` (both through `score_round`) |
| Loop decision | Key `run:<label>:cycle:<k>`; type `weksa.loop_decision` with the base round and revision, the edit prompt's digest and Train keys, the editor's answer digest, the candidate revision, inventory digest and round, the base, candidate and floor readings (chrF, meaning, composite), the threshold and the verdict (Accepted, or Rejected for edit-parse, edit-admit, no-gain, regressed chrF, meaning or composite); digests and numbers, no text | Immutable. The latest Accepted one names the run's head inventory (`<root>/cycle-<k>/inventory.cc`, its candidate digest, its candidate round); with none the head is the copy `create` made at `<root>/inventory.cc`. Lives in the run store on Yggdrasil | `run::cycle`'s accept rule, `run::decide`: the validation composite gains more than max(noise floor, `LADDER_STEP`), no metric falls past max(its floor, one step), the composite does not fall past max(its floor, one step), and a non-finite reading rejects (the one regression test, `run::regression`); differences compared in billionths so a gain of exactly one step is a tie. From cut `ja-panel-loop` the readings, floor and threshold are kept per panel language and `run::regression` runs per language and is the one regression test (`decide` adds the gain test for a one-language panel): any language regressing rejects (verdicts `regressed-chrf`, `regressed-meaning`, `regressed-composite`), otherwise a one-language panel accepts as `decide` does and a panel of two or more accepts when the sum of composite deltas exceeds one `LADDER_STEP` (billionths). Given up: strict Pareto non-regression; a language may lose up to its own noise floor if the panel nets more than a step, because zero tolerance ratchets every base to a favourable noise draw and strangles the loop. Termination: each accept raises the composite sum by more than a step and composites lie in -0.5 to 1, so a run accepts at most 1.5 * languages / `LADDER_STEP` edits |
| Score | Key `<lowering-key>:<metric>@<metric-version>`; type `weksa.score` | Immutable; a new metric version scores again under a new key | The scorer, deterministic given the metric version and the embedder's model id. Cut `scorer` computes scores as values; cut `golf-loop-en` stores them, because the lowering keys they hang from arrive with the loop. Lands with follow-up `golf-loop-score-docs`, in the round's directory; nothing in the cycle reads them |
| Lowering trace | Key `lowering:<packet-key>:<profile>@<rev>:<lowerer-digest>:<sample>`; type `weksa.lowering` holding the surface text and its trace together | Immutable. A surface text cannot exist without its trace, because they are one record (`lowering-traced`) | The lowerer emits; the lowerer sees the packet without its `trace` (the input type has no such field), never the corpus item |

50
schemas/cultnet/weksa.loop_round.schema.json
View file

@ -1,7 +1,7 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "RoundRecord",
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::RoundRecord;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n languages: vec![],\n }\n}\n```",
"description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::RoundRecord;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n languages: vec![],\n validation_scores: vec![],\n }\n}\n```",
"type": "object",
"properties": {
"admit_rules": {
@ -87,6 +87,14 @@
"items": {
"$ref": "#/$defs/TrainRow"
}
},
"validation_scores": {
"description": "Each panel language's live Validation scores in corpus order, a refused\nitem as 0. Private with no getter: only `run.rs` (the floor) reads them,\nso a leak outside it is a compile error. A run from before has none.",
"type": "array",
"default": [],
"items": {
"$ref": "#/$defs/LanguageScores"
}
}
},
"required": [
@ -123,6 +131,27 @@
"items"
]
},
"ItemScore": {
"type": "object",
"properties": {
"chrf": {
"description": "chrF of the output against the reference.",
"type": "number",
"format": "double"
},
"meaning": {
"description": "Embedding cosine of the output and the reference; `None` when either\nis empty after trimming.",
"type": [
"number",
"null"
],
"format": "double"
}
},
"required": [
"chrf"
]
},
"LanguageRound": {
"description": "One panel language's readings in a round: the means over that language's\nitems only (a refused item counts 0) and how many items lowered and scored.",
"type": "object",
@ -161,6 +190,25 @@
"validation_refused"
]
},
"LanguageScores": {
"description": "One panel language's Validation scores in a round, in corpus order: numbers\nonly, no item key and no text. Only `run.rs` reads them (the paired floor).",
"type": "object",
"properties": {
"language": {
"type": "string"
},
"scores": {
"type": "array",
"items": {
"$ref": "#/$defs/ItemScore"
}
}
},
"required": [
"language",
"scores"
]
},
"Outcome": {
"oneOf": [
{