From 17f44144a5372ec80eefd318c31a8def35986b27 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sun, 8 Nov 2026 00:50:39 +0000 Subject: [PATCH 01/84] Show her working: static page of the lowering chain from the real lowerer and scorer Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 1 + crates/weksa/examples/show_working.rs | 373 ++++++++++++++++++++++++++ scripts/publish-show-working.sh | 16 ++ tools/show-working/README.md | 9 + tools/show-working/build.py | 203 ++++++++++++++ 5 files changed, 602 insertions(+) create mode 100644 crates/weksa/examples/show_working.rs create mode 100755 scripts/publish-show-working.sh create mode 100644 tools/show-working/README.md create mode 100755 tools/show-working/build.py diff --git a/.gitignore b/.gitignore index 1428a57..e00572b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ build/ .venv/ venv/ target/ +tools/show-working/out/ diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs new file mode 100644 index 0000000..cf24b64 --- /dev/null +++ b/crates/weksa/examples/show_working.rs @@ -0,0 +1,373 @@ +//! Build-time generator for the "Show her working" page. Runs the real +//! admission door, lowerer and IPA reader over the committed proof packet and +//! variants of it, and prints one JSON document on stdout: the typed slots, the +//! inventory entries the packet chose, and the outcome, which is lowered text or +//! a named refusal. Scoring is not here: `tools/show-working/build.py` calls the +//! `weksa score` CLI on the text this emits. +//! +//! Run from the repository root: `cargo run --release --example show_working`. +//! Every variant is a mutation of `proof_packet()`, spelled in its `source`. + +use std::path::Path; + +use serde_json::{Value, json}; +use weksa::packet::{ + EntityId, FeatValue, FeatureValue, Grade, NodeId, ProvenanceSource, RoleFill, proof_packet, +}; +use weksa::{ + CorpusStore, EntityStore, InvId, Inventory, InventoryStore, LowerError, Packet, PacketStore, + StoreError, lower, pronounce, +}; + +struct Fixture { + id: &'static str, + title: &'static str, + note: &'static str, + source: &'static str, + packet: Packet, +} + +fn id(s: &str) -> InvId { + s.parse().expect("inventory id") +} + +fn node(s: &str) -> NodeId { + NodeId::parse(s).expect("node id") +} + +fn grade(feature: &str, v: u16) -> FeatureValue { + FeatureValue { + feature: id(feature), + value: FeatValue::Grade(Grade::new(v).expect("grade")), + } +} + +fn fixtures() -> Vec { + let proof = proof_packet(); + let mut out = vec![Fixture { + id: "proof", + title: "Nibu warns the player off the control", + note: "The committed proof packet, unchanged.", + source: "proof_packet() in crates/weksa/src/packet.rs", + packet: proof.clone(), + }]; + let mut add = |id, title, note, source, f: &dyn Fn(&mut Packet)| { + let mut packet = proof.clone(); + f(&mut packet); + out.push(Fixture { id, title, note, source, packet }); + }; + add( + "no-addressee", + "Same meaning, nobody addressed", + "With the addressee unset the player is a third party: no \"you\", no reflexive.", + "proof_packet() with discourse.addressee = None", + &|p| p.discourse.addressee = None, + ); + add( + "stance-held-back", + "Anger held back, restraint high", + "Stance is the speaker's felt affect. The lowerer has no row that reads it, so the text is the proof's text.", + "proof_packet() with stance anger=300, restraint=700", + &|p| p.stance.features = vec![grade("feature:9", 300), grade("feature:18", 700)], + ); + add( + "hedged", + "Hedged and uncertain warning", + "Hedge and certainty are meaning the lowerer cannot yet say. It refuses rather than drop them.", + "proof_packet() with discourse hedge=600, certainty=400", + &|p| p.discourse.features = vec![grade("feature:6", 600), grade("feature:7", 400)], + ); + add( + "negated", + "Negated intervention (polarity: negative)", + "Dropping the negation would say the opposite, so the lowerer refuses. This is the shape of a negated Dao line awaiting polarity.", + "proof_packet() with predication n5 polarity=negative", + &|p| { + p.predications[1].features = vec![FeatureValue { + feature: id("feature:1"), + value: FeatValue::Option("negative".parse().expect("label")), + }] + }, + ); + add( + "consent-authored", + "Addressee consent, authored source", + "Someone who saw the request says the addressee asked for it. No lowering speaks for that request yet.", + "proof_packet() with discourse.addressee_consent = n4 (source Authored)", + &|p| p.discourse.addressee_consent = Some(node("n4")), + ); + add( + "consent-imported", + "Addressee consent, imported-corpus source", + "A reader of wording cannot have seen a request, so the packet is refused at admission (V15), before any lowering.", + "proof_packet() with addressee_consent = n4 and provenance.source = ImportedCorpus", + &|p| { + p.discourse.addressee_consent = Some(node("n4")); + p.provenance.source = ProvenanceSource::ImportedCorpus; + }, + ); + add( + "relation", + "A causal relation between the predications", + "Relations are not rendered; the lowerer refuses instead of asserting a different meaning.", + "proof_packet() with relation:8 (causes) n4 -> n5", + &|p| { + p.relations = vec![weksa::packet::Relation { + relation: id("relation:8"), + from: node("n4"), + to: node("n5"), + features: vec![], + }] + }, + ); + add( + "dangling-filler", + "A role filled by a node that does not exist", + "Refused at admission (V4): the packet is not a packet.", + "proof_packet() with predication n5 agent filler n9", + &|p| { + p.predications[1].roles[0] = RoleFill { role: id("role:1"), filler: node("n9") } + }, + ); + add( + "unknown-entity", + "A referent naming an entity nobody holds", + "The entity store is the only owner of names; a packet cannot invent one.", + "proof_packet() with referent n3 entity = entity:nobody", + &|p| p.referents[2].entity = Some(EntityId::parse("entity:nobody").expect("entity")), + ); + add( + "common-noun-agent", + "The ship mind as a common noun", + "No entity: the referent lowers as \"the ship mind\", and there is no name to pronounce.", + "proof_packet() with referent n3 entity = None", + &|p| p.referents[2].entity = None, + ); + out +} + +fn gloss(inv: &Inventory, rev: u32, i: &InvId) -> Value { + match inv.resolve(i, rev) { + Some(e) => json!({ "id": i.to_string(), "gloss": e.gloss }), + None => json!({ "id": i.to_string(), "gloss": null }), + } +} + +fn feature_json(inv: &Inventory, rev: u32, slot: &str, at: &str, f: &FeatureValue) -> Value { + let mut g = gloss(inv, rev, &f.feature); + g["slot"] = json!(slot); + g["at"] = json!(at); + g["value"] = serde_json::to_value(&f.value).expect("value"); + g +} + +/// Every feature the packet carries, with where it sits. +fn features(p: &Packet, inv: &Inventory) -> Vec { + let rev = p.inventory_rev; + let mut v = Vec::new(); + for f in &p.discourse.features { + v.push(feature_json(inv, rev, "discourse", "discourse", f)); + } + for f in &p.stance.features { + v.push(feature_json(inv, rev, "stance", "stance", f)); + } + for r in &p.referents { + for f in &r.features { + v.push(feature_json(inv, rev, "referent", r.id.as_str(), f)); + } + } + for pr in &p.predications { + for f in &pr.features { + v.push(feature_json(inv, rev, "predication", pr.id.as_str(), f)); + } + } + v +} + +/// Ids the packet chose, once each, with the inventory's gloss at its revision. +fn chosen(p: &Packet, inv: &Inventory) -> Vec { + let rev = p.inventory_rev; + let mut ids: Vec = Vec::new(); + let mut push = |i: &InvId| { + if !ids.contains(i) { + ids.push(*i) + } + }; + p.context.setting.iter().for_each(&mut push); + p.context.medium.iter().for_each(&mut push); + p.discourse.speech_act.iter().for_each(&mut push); + for r in &p.referents { + push(&r.kind); + push(&r.concept); + } + for pr in &p.predications { + push(&pr.concept); + push(&pr.frame); + pr.roles.iter().for_each(|f| push(&f.role)); + } + p.relations.iter().for_each(|r| push(&r.relation)); + features(p, inv) + .iter() + .filter_map(|f| f["id"].as_str().and_then(|s| s.parse().ok())) + .for_each(|i: InvId| push(&i)); + ids.iter().map(|i| gloss(inv, rev, i)).collect() +} + +fn slots(p: &Packet) -> Value { + let j = |v: &dyn erased::Ser| v.to_value(); + json!({ + "meaning": { + "referents": p.referents.iter().map(|r| j(r)).collect::>(), + "predications": p.predications.iter().map(|r| j(r)).collect::>(), + "relations": p.relations.iter().map(|r| j(r)).collect::>(), + }, + "stance": j(&p.stance), + "discourse": j(&p.discourse), + "context": j(&p.context), + "extensions": p.extensions.iter().map(|r| j(r)).collect::>(), + "constraints": j(&p.constraints), + "provenance": { "source": j(&p.provenance.source), "corpus_item": p.provenance.corpus_item.to_string(), "confidence": p.provenance.confidence.get() }, + "kind": j(&p.kind), + "inventory_rev": p.inventory_rev, + }) +} + +mod erased { + pub trait Ser { + fn to_value(&self) -> serde_json::Value; + } + impl Ser for T { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(self).expect("serialize") + } + } +} + +fn lower_error(e: &LowerError) -> Value { + let (variant, reason) = match e { + LowerError::Unrendered { what } => ("Unrendered", format!("the packet carries {what}, which the lowerer does not render")), + LowerError::NoRow { id } => ("NoRow", format!("no table row for {id}")), + LowerError::GlossMismatch { id, row } => ("GlossMismatch", format!("{id} is not the {row} the row speaks for")), + LowerError::UnknownRole { frame, role } => ("UnknownRole", format!("{frame} names role {role}, which the inventory lacks")), + LowerError::MissingRole { predication, role } => ("MissingRole", format!("{predication} fills no {role} role")), + LowerError::UnknownEntity { entity } => ("UnknownEntity", format!("the entity store does not hold {entity}")), + LowerError::NoForm { entity } => ("NoForm", format!("{entity} has no en-US form")), + LowerError::NoPronunciation { entity } => ("NoPronunciation", format!("{entity} has no ruled en-US pronunciation")), + LowerError::NoEntityReferent => ("NoEntityReferent", "the packet cites no entity".to_string()), + LowerError::EntityRead { entity, why } => ("EntityRead", format!("reading {entity}: {why}")), + }; + json!({ "variant": variant, "reason": reason, "debug": format!("{e:?}") }) +} + +struct World { + inventory: InventoryStore, + corpus: CorpusStore, + entities: EntityStore, + inv: Inventory, +} + +/// Admit `packet` through a fresh packet store and lower it. `Ok` carries the +/// text and the IPA outcome; `Err` is the refusal, as JSON. +fn run(w: &World, dir: &Path, packet: &Packet) -> Result<(String, Value), Value> { + let mut store = PacketStore::open(&dir.join("p.cc")).expect("packet store"); + let view = match store.admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) { + Ok(v) => v, + Err(StoreError::Invalid(rs)) => { + return Err(json!({ + "kind": "admission_refused", + "refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": format!("{r:?}") })).collect::>(), + })); + } + Err(e) => { + return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": format!("{e:?}") }] })); + } + }; + let text = lower(&view, &w.inv, &w.entities).map_err(|e| { + json!({ "kind": "lower_error", "error": lower_error(&e) }) + })?; + let ipa = match pronounce(&view, &w.entities) { + Ok(v) => json!({ "ok": v }), + Err(e) => json!({ "error": lower_error(&e) }), + }; + Ok((text, ipa)) +} + +fn main() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).expect("inventory"); + let inv = inventory.inventory().expect("inventory view"); + let w = World { + corpus: CorpusStore::open(&root.join("data/corpus/corpus.cc")).expect("corpus"), + entities: EntityStore::open(&root.join("data/entity/entity.cc")).expect("entities"), + inventory, + inv, + }; + let dir = std::env::temp_dir().join(format!("show-working-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("scratch dir"); + let mut out = Vec::new(); + for fx in fixtures() { + let p = &fx.packet; + let first = run(&w, &dir, p); + let outcome = match first { + Err(e) if e["kind"] == "admission_refused" => e, + first => { + // Ablation: run again with each feature removed, against the + // baseline outcome. It is how the page says whether the lowerer + // read a feature, instead of the page asserting it. + let base = first.as_ref().ok().map(|(t, _)| t.clone()); + let n_disc = p.discourse.features.len(); + let n_stance = p.stance.features.len(); + let total = features(p, &w.inv).len(); + let mut effects = Vec::new(); + for k in 0..total { + let mut q = p.clone(); + if k < n_disc { + q.discourse.features.remove(k); + } else if k < n_disc + n_stance { + q.stance.features.remove(k - n_disc); + } else { + let mut at = k - n_disc - n_stance; + 'o: { + for r in &mut q.referents { + if at < r.features.len() { r.features.remove(at); break 'o; } + at -= r.features.len(); + } + for pr in &mut q.predications { + if at < pr.features.len() { pr.features.remove(at); break 'o; } + at -= pr.features.len(); + } + } + } + let _ = std::fs::remove_file(dir.join("p.cc")); + effects.push(match (&base, run(&w, &dir, &q)) { + (Some(t), Ok((u, _))) if *t == u => "Without it the text is identical: the lowerer did not read it.", + (Some(_), Ok(_)) => "Without it the text changes: the lowerer read it.", + (Some(_), Err(_)) => "Without it the packet is refused.", + (None, Ok(_)) => "Without it the packet lowers: this feature is what the lowerer refuses.", + (None, Err(_)) => "Still refused without this feature alone.", + }); + } + match first { + Ok((text, ipa)) => json!({ "kind": "lowered", "text": text, "ipa": ipa, "feature_effects": effects }), + Err(e) => { let mut e = e; e["feature_effects"] = json!(effects); e } + } + } + }; + let _ = std::fs::remove_file(dir.join("p.cc")); + let mut slots = slots(p); + slots["features"] = json!(features(p, &w.inv)); + out.push(json!({ + "id": fx.id, + "title": fx.title, + "note": fx.note, + "source": fx.source, + "packet_id": p.id.to_string(), + "corpus_item": p.provenance.corpus_item.to_string(), + "slots": slots, + "chosen": chosen(p, &w.inv), + "outcome": outcome, + })); + } + let _ = std::fs::remove_dir_all(&dir); + println!("{}", serde_json::to_string_pretty(&json!({ "fixtures": out })).expect("json")); +} diff --git a/scripts/publish-show-working.sh b/scripts/publish-show-working.sh new file mode 100755 index 0000000..3af3194 --- /dev/null +++ b/scripts/publish-show-working.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Build the "Show her working" page from the real lowerer and scorer, then publish it as a Mind preview. +# Run from the repository root, in a Mind's Self run (it writes only /srv/minds-web//). +# +# scripts/publish-show-working.sh [--no-publish] +# +# Needs cargo, python3 and an Ollama embedder at 127.0.0.1:11434 (qwen3-embedding:0.6b): the scorer is not stubbed. +# publish-mind-preview.sh lives in the gamecult-ops checkout: set GAMECULT_OPS to it (default ../gamecult-ops). +set -euo pipefail +[ -f Cargo.toml ] && [ -d tools/show-working ] || { echo "run from the Weksa repository root" >&2; exit 2; } +out=${SHOW_WORKING_OUT:-tools/show-working/out} +rm -rf "$out" +python3 -I tools/show-working/build.py "$out" +[ "${1:-}" = --no-publish ] && { echo "built $out"; exit 0; } +ops=${GAMECULT_OPS:-../gamecult-ops} +exec "$ops/scripts/publish-mind-preview.sh" "$out" weksa-show-working diff --git a/tools/show-working/README.md b/tools/show-working/README.md new file mode 100644 index 0000000..5e2a320 --- /dev/null +++ b/tools/show-working/README.md @@ -0,0 +1,9 @@ +# Show her working + +A static page that shows the lowering chain for committed fixture packets: typed slots, inventory entries chosen, lowered English, scorer closeness to the corpus target, with hedge, certainty and restraint annotated. Refusals (admission rules, `LowerError`, addressee consent, negation awaiting polarity) are results, not omissions. + +- `crates/weksa/examples/show_working.rs`: runs the real admission door, `lower` and `pronounce` over `proof_packet()` and its variants and prints JSON. Each feature is removed in turn and the packet re-run, so "the lowerer read this" is measured. +- `build.py`: runs that, scores each lowered text with the `weksa score` CLI (needs Ollama with `qwen3-embedding:0.6b`), writes `out/data.json` and one dependency-free `out/index.html`. Fails if anything cannot run; there is no fallback text. +- `scripts/publish-show-working.sh`: build, then publish through gamecult-ops `publish-mind-preview.sh`. + +Adding a fixture: add an entry in `fixtures()` in the example. A packet from another store needs a loader there; none exists yet. diff --git a/tools/show-working/build.py b/tools/show-working/build.py new file mode 100755 index 0000000..269032e --- /dev/null +++ b/tools/show-working/build.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Build the "Show her working" page from the real lowerer and scorer. + +usage: build.py [out_dir] (run from the repository root) + +1. `cargo run --example show_working` admits each fixture packet through the + packet store and lowers it with the English lowerer (crates/weksa/examples). +2. `weksa score` scores each lowered text against its corpus item (and the + other committed Nibu reference) through the Ollama embedder. +3. out_dir/data.json holds the joined result; out_dir/index.html is one file + with no scripts to fetch and no network calls. + +Nothing on the page is typed by hand except the layout and the reading notes. +If cargo or the scorer cannot run, the build fails: there is no fallback text. +Standard library only. +""" +import json +import pathlib +import re +import subprocess +import sys +import time +from html import escape as e + +root = pathlib.Path.cwd() +out_dir = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "tools/show-working/out") +weksa_bin = None + + +def run(cmd, **kw): + r = subprocess.run(cmd, cwd=root, capture_output=True, text=True, **kw) + if r.returncode != 0: + sys.exit(f"{' '.join(map(str, cmd))} failed:\n{r.stderr}") + return r.stdout + + +def cargo_target(): + meta = json.loads(run(["cargo", "metadata", "--format-version", "1", "--no-deps"])) + return pathlib.Path(meta["target_directory"]) + + +# -- 1. the real lowerer ------------------------------------------------------ +run(["cargo", "build", "--release", "--bin", "weksa", "--example", "show_working"]) +target = cargo_target() / "release" +weksa_bin = target / "weksa" +data = json.loads(run([str(target / "examples" / "show_working")])) + +# -- 2. the real scorer ------------------------------------------------------- +corpus = {} +for m in re.finditer(r'^(item:\S+) \[(.*?)\]\n "(.*)"$', run([str(weksa_bin), "corpus", "dump"]), re.M): + corpus[m.group(1)] = {"text": m.group(3), "meta": m.group(2)} +REFS_EXTRA = ["item:nibu:2"] +model = "qwen3-embedding:0.6b" + + +def score(text, items): + out = run([str(weksa_bin), "score", *items], input=text) + got = {} + for m in re.finditer(r"^(item:\S+) chrf=([\d.]+) meaning=(-?[\d.]+)", out, re.M): + got[m.group(1)] = {"chrf": float(m.group(2)), "meaning": float(m.group(3))} + if set(got) != set(items): + sys.exit(f"scorer answered {sorted(got)} for {items}:\n{out}") + return got + + +for fx in data["fixtures"]: + o = fx["outcome"] + if o["kind"] == "lowered": + refs = [fx["corpus_item"]] + [r for r in REFS_EXTRA if r != fx["corpus_item"]] + o["scores"] = [{"item": k, "target": k == fx["corpus_item"], **v} for k, v in score(o["text"], refs).items()] + +data["corpus"] = corpus +data["build"] = { + "commit": run(["git", "rev-parse", "HEAD"]).strip(), + "embedder": model, + "built_unix": int(time.time()), +} +out_dir.mkdir(parents=True, exist_ok=True) +(out_dir / "data.json").write_text(json.dumps(data, indent=1, ensure_ascii=False), encoding="utf-8") + +# -- 3. the page -------------------------------------------------------------- +CSS = """ +:root{--ink:#1c1b19;--mute:#5d5a54;--line:#cfc9bd;--bg:#f6f3ec;--card:#fff;--ok:#0b6b4b;--warn:#9a4a00;--bad:#9b1c2c;--bar:#365f9c} +*{box-sizing:border-box}body{font:16px/1.55 system-ui,sans-serif;max-width:62rem;margin:2rem auto;padding:0 1rem;color:var(--ink);background:var(--bg)} +h1{margin:0 0 .2rem}h2{margin:0 0 .4rem}h3{margin:1.2rem 0 .3rem;font-size:.85rem;text-transform:uppercase;letter-spacing:.06em;color:var(--mute)} +code,.mono{font:13px/1.4 ui-monospace,Menlo,monospace}code{background:#ece8dd;padding:0 .25rem;border-radius:3px} +.lede{max-width:46rem}.meta{color:var(--mute);font-size:.88rem} +nav.pick{display:flex;flex-wrap:wrap;gap:.4rem;margin:1.2rem 0} +nav.pick a{border:1px solid var(--line);background:var(--card);padding:.25rem .6rem;border-radius:999px;text-decoration:none;color:var(--ink);font-size:.9rem} +nav.pick a.refused{border-color:var(--bad)}nav.pick a.lowered{border-color:var(--ok)} +article{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:1rem 1.3rem;margin:1.4rem 0} +.chain{display:grid;grid-template-columns:repeat(4,1fr);gap:.6rem;margin:.6rem 0 1rem} +.chain div{border:1px solid var(--line);border-radius:6px;padding:.3rem .6rem;font-size:.82rem;color:var(--mute)} +.chain b{display:block;color:var(--ink)} +table{border-collapse:collapse;width:100%;font-size:.9rem}th,td{text-align:left;padding:.25rem .5rem;border-bottom:1px solid #e6e1d5;vertical-align:top}th{color:var(--mute);font-weight:600} +.out{font-size:1.15rem;padding:.6rem .9rem;border-left:4px solid var(--ok);background:#eef6f1} +.refusal{padding:.6rem .9rem;border-left:4px solid var(--bad);background:#faeef0}.refusal b{color:var(--bad)} +.tag{display:inline-block;font-size:.75rem;padding:0 .4rem;border-radius:3px;border:1px solid var(--line);color:var(--mute)} +.tag.read{border-color:var(--ok);color:var(--ok)}.tag.unread{border-color:var(--warn);color:var(--warn)}.tag.refuse{border-color:var(--bad);color:var(--bad)} +.bar{height:.7rem;background:#e4dfd2;border-radius:3px;min-width:8rem;position:relative}.bar i{display:block;height:100%;background:var(--bar);border-radius:3px} +.ref{font-size:.88rem;color:var(--mute);margin:.2rem 0 .6rem}.ref q{color:var(--ink)} +details{margin:.5rem 0}summary{cursor:pointer;color:var(--mute);font-size:.88rem} +section.gaps li{margin:.3rem 0} +@media(max-width:700px){.chain{grid-template-columns:1fr 1fr}} +""" + +PICK_JS = """(function(){var a=[].slice.call(document.querySelectorAll('article[data-fx]'));function show(){var h=location.hash.slice(1);var any=a.some(function(x){return x.id===h});a.forEach(function(x){x.hidden=any&&x.id!==h})} +window.addEventListener('hashchange',show);show()})()""" + + +def g(chosen, i): + return next((c["gloss"] for c in chosen if c["id"] == i), "?") + + +def val(v): + (k, x), = v.items() + return f"{x}‰" if k == "Grade" else str(x) + + +def slots_html(fx): + s, ch = fx["slots"], fx["chosen"] + rows = [] + for r in s["meaning"]["referents"]: + ent = f" · entity {e(r['entity'])}" if r["entity"] else "" + rows.append(f"{e(r['id'])}referent{e(g(ch, r['kind']))} / {e(g(ch, r['concept']))}{ent}") + for p in s["meaning"]["predications"]: + roles = ", ".join(f"{e(g(ch, x['role']))}={e(x['filler'])}" for x in p["roles"]) + rows.append(f"{e(p['id'])}predication{e(g(ch, p['concept']))} in frame {e(g(ch, p['frame']))}: {roles}") + for r in s["meaning"]["relations"]: + rows.append(f"relation{e(g(ch, r['relation']))}: {e(r['from'])} → {e(r['to'])}") + meaning = "" + "".join(rows) + "
nodekindcontent
" + d = s["discourse"] + drows = [] + if d["speech_act"]: + drows.append(f"speech act{e(g(ch, d['speech_act']))}") + if d["addressee"]: + drows.append(f"addressee{e(d['addressee'])}") + if d["addressee_consent"]: + drows.append(f"addressee consentpredication {e(d['addressee_consent'])} (what the addressee asked for)") + drows.append(f"source{e(s['provenance']['source'])}, kind {e(s['kind'])}") + return (f"

Slot: meaning

{meaning}

Slot: discourse

{''.join(drows)}
" + + stance_register(fx)) + + +def stance_register(fx): + s = fx["slots"] + eff = fx["outcome"].get("feature_effects") or [] + rows = [] + for i, f in enumerate(s["features"]): + note = eff[i] if i < len(eff) else "" + cls = "read" if "read it" in note else "unread" if "did not read" in note else "refuse" if "refus" in note else "" + rows.append(f"{e(f['gloss'])} {e(f['id'])}{e(f['slot'])} @ {e(f['at'])}{e(val(f['value']))}{e(note)}") + feats = ("" + "".join(rows) + "
featureslotvaluemeasured effect on the lowerer
") if rows else "

No features: no hedge, certainty, polarity, irony or stance set.

" + return ("

Slot: stance, hedge, certainty, restraint (features)

" + feats + + "

Register: the packet has no register slot. Register and address belong to a target-language profile, not to the packet; this lowerer reads none.

") + + +def outcome_html(fx): + o = fx["outcome"] + if o["kind"] == "lowered": + ipa = o["ipa"] + ipa_s = ("IPA of cited entities: " + e(" ".join(ipa["ok"])) + "") if "ok" in ipa else f"IPA refused: {e(ipa['error']['reason'])} {e(ipa['error']['variant'])}" + sc = [] + for s in o["scores"]: + ref = corpus[s["item"]]["text"] + sc.append(f"
{'Target' if s['target'] else 'Other reference'} {e(s['item'])}: {e(ref)}
" + f"" + f"
chrF {s['chrf']:.1f}
meaning cosine {s['meaning']:.3f}
") + return (f"

Lowered English

{e(o['text'])}

{ipa_s}

" + f"

Closeness to the corpus ({e(data['build']['embedder'])})

{''.join(sc)}") + if o["kind"] == "lower_error": + er = o["error"] + return f"

Refused by the lowerer

LowerError::{e(er['variant'])}: {e(er['reason'])}
{e(er['debug'])}

No text, so no score: nothing was invented to score.

" + items = "".join(f"
  • {e(r['rule'])} {e(r['detail'])}
    • " for r in o["refusals"]) + return f"

    Refused at admission

      {items}

    The packet never became a lowering input, so the lowerer never saw it.

    " + + +def card(fx): + o = fx["outcome"] + ch = "".join(f"{e(c['id'])}{e(c['gloss'] or '(unresolved)')}" for c in fx["chosen"]) + steps = [("1 Typed slots", "meaning, stance, discourse"), ("2 Inventory entries", f"{len(fx['chosen'])} ids, rev {fx['slots']['inventory_rev']}"), + ("3 Lowered English", {"lowered": "text", "lower_error": "refused", "admission_refused": "never reached"}[o["kind"]]), + ("4 Closeness", "scored" if o["kind"] == "lowered" else "no text, no score")] + chain = "".join(f"
    {e(a)}{e(b)}
    " for a, b in steps) + return (f"

    {e(fx['title'])}

    {e(fx['note'])}

    " + f"

    Packet {e(fx['packet_id'])} for {e(fx['corpus_item'])}. Made from: {e(fx['source'])}.

    " + f"
    {chain}
    {slots_html(fx)}" + f"
    Inventory entries chosen ({len(fx['chosen'])}){ch}
    idgloss at this revision
    " + f"{outcome_html(fx)}
    ") + + +nav = "".join(f"{e(f['title'])}" for f in data["fixtures"]) +b = data["build"] +page = f""" +Weksa: show her working +

    Show her working

    +

    A meaning packet goes in. The English lowerer picks inventory entries, writes a sentence, and a scorer measures how close that sentence is to a human-written target. Every number and sentence below was produced by running the real Rust lowerer and scorer when this page was built. The refusals are results too: when the lowerer cannot say a meaning without changing it, it says so instead of dropping the part it cannot say.

    +

    Built from {e(b['commit'][:10])}, embedder {e(b['embedder'])}. English only. Fixtures are the committed nibu-proof-001 packet and variants of it, each variant described in its own card. Scores are against the committed corpus: chrF (0-100) and embedding cosine. Raw data: data.json.

    + +{''.join(card(f) for f in data['fixtures'])} +""" +(out_dir / "index.html").write_text(page, encoding="utf-8") +print(f"wrote {out_dir}/index.html and data.json ({len(data['fixtures'])} fixtures)") From 2259b9f8c3fc4bd48deb4ca13a4b9cd9b822b66f Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:26:14 +0000 Subject: [PATCH 02/84] WIP (does not build): DraftClass on DraftError; create() call sites take a ModelFamily argument Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 62 +++++++++++++++++++++++++++ crates/weksa/src/run.rs | 2 +- crates/weksa/tests/golf_loop.rs | 28 ++++++------ crates/weksa/tests/original_corpus.rs | 2 +- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 5071316..2efb331 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -47,6 +47,67 @@ pub struct PacketDraft { pub struct DraftError { pub line: usize, pub column: usize, + /// What kind of reading failure, from serde's own fixed vocabulary. It + /// never holds a token of the answer. + pub class: DraftClass, +} + +/// Why an answer is not a draft: a closed set of classes, never the offending +/// text, so a record may keep it for a held-out item. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DraftClass { + /// Not JSON. + Syntax, + /// JSON that ends early. + Eof, + MissingField, + UnknownField, + /// A value of the wrong JSON type, such as a bare integer for an id. + InvalidType, + /// A value of the right type that is not one of the field's: a node id + /// spelled `entity:x`, a number out of range. + InvalidValue, + Other, +} + +impl DraftClass { + /// The code a round record keeps. + pub fn code(self) -> &'static str { + match self { + DraftClass::Syntax => "syntax", + DraftClass::Eof => "eof", + DraftClass::MissingField => "missing-field", + DraftClass::UnknownField => "unknown-field", + DraftClass::InvalidType => "invalid-type", + DraftClass::InvalidValue => "invalid-value", + DraftClass::Other => "other", + } + } + + fn of(e: &serde_json::Error) -> Self { + use serde_json::error::Category; + match e.classify() { + Category::Syntax => DraftClass::Syntax, + Category::Eof => DraftClass::Eof, + Category::Io => DraftClass::Other, + // Only the leading words of serde's message are read, and only to + // pick one of these classes; the message is not kept. + Category::Data => { + let msg = e.to_string(); + if msg.starts_with("missing field") { + DraftClass::MissingField + } else if msg.starts_with("unknown field") { + DraftClass::UnknownField + } else if msg.starts_with("invalid type") { + DraftClass::InvalidType + } else if msg.starts_with("invalid value") { + DraftClass::InvalidValue + } else { + DraftClass::Other + } + } + } + } } impl fmt::Display for DraftError { @@ -72,6 +133,7 @@ fn from_json(answer: &str) -> Result { serde_json::from_str(answer.trim()).map_err(|e| DraftError { line: e.line(), column: e.column(), + class: DraftClass::of(&e), }) } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index bcf0d00..517101d 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2059,7 +2059,7 @@ mod tests { entities: &entities, }, "fake-model", - "bag", crate::run::LowererReach::ToolLessRelay, + "bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); let mut model = fake(); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 4d9dc1c..377e8f3 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -47,7 +47,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { entities: &e, }, model, - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; @@ -148,7 +148,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { entities: &e, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); @@ -223,7 +223,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; @@ -268,7 +268,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; // A directory opens for reading and fails mid-copy, after the destination exists. @@ -381,14 +381,14 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { entities: &e, }; assert_eq!( - RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || more(&mut w)).err(), + RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(), Some(RunError::Drift(DriftOf::Corpus)) ); assert!(!root.join("inventory.cc").exists()); // Not the copy and not the lock the store's open left beside it. assert!(!root.join("inventory.cc.lock").exists()); assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun)); - RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay).unwrap(); + RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap(); } /// Runs `change` in the window between the pins and the open, at `open` and at @@ -411,7 +411,7 @@ fn window( inventory: &i, entities: &e, }; - let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || { + let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || { change(&mut w, &root) }) .err(); @@ -828,7 +828,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao { entities: &entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); Dao { @@ -1477,15 +1477,15 @@ fn errors_echo_no_input() { inventory: &missing, entities: &missing, }; - let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); - let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); - let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); @@ -1519,7 +1519,7 @@ fn errors_echo_no_input() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); let mut model = cycle_fake(&format!("{{\"{canary}\":1}}")); @@ -2449,7 +2449,7 @@ fn a_changed_head_is_drift() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); assert_eq!( @@ -2768,7 +2768,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { inventory: &dao.inventory, entities: &dao.entities, }; - RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent) + RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded) .unwrap(); let run = RunStores::open(&root).unwrap(); assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent); diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 51e4c35..bbf7056 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -99,7 +99,7 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { }, "m", "bag", - weksa::run::LowererReach::BriefedSubagent, + weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded ) .unwrap(); let mut model = Unrelated(vec![]); From 2759c0f12d651aa2890b84c63304ab164b28d67e Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 17:05:07 +0000 Subject: [PATCH 03/84] Round records carry a closed reason for every parse refusal and an own-family count DraftClass (closed set from serde's category and leading words, never the message) rides on DraftError; the round counts parse_reasons by code ('unreadable' for an answer that did not read as text) beside admit_rules. RunRecord keeps the answerer's ModelFamily (loop open --answerer-family); the round counts offered ModelWritten items of that family and the line says own-family N. The corpus rule is documented as binding model judges: the loop labels, it does not forbid. New fields and admit_rules are cultcache default, so older run.cc records decode; schemas regenerated. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 12 +- crates/weksa/src/corpus.rs | 8 +- crates/weksa/src/encode.rs | 35 ++- crates/weksa/src/main.rs | 6 + crates/weksa/src/run.rs | 88 +++++- crates/weksa/tests/cli.rs | 4 +- crates/weksa/tests/common/mod.rs | 2 +- crates/weksa/tests/golf_loop.rs | 277 ++++++++++++++++++ crates/weksa/tests/original_corpus.rs | 76 +++++ schemas/cultnet/weksa.corpus_item.schema.json | 2 +- schemas/cultnet/weksa.loop_round.schema.json | 17 +- schemas/cultnet/weksa.loop_run.schema.json | 26 ++ 12 files changed, 528 insertions(+), 25 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 3c73b0c..5592e1e 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -86,7 +86,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// assert!(t.is_empty()); /// ``` @@ -100,7 +100,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let _u = t.clone(); /// ``` @@ -111,7 +111,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// s(&t); /// ``` @@ -317,7 +317,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -335,7 +335,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -348,7 +348,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 2604c6a..a2f94ae 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -43,7 +43,10 @@ pub enum Source { translator: String, edition_year: u16, }, - /// Labelled model output; never in the test split, never judged by its own family. + /// Labelled model output; never in the test split. The rule that it is + /// never judged by its own family binds model judges and the split; a + /// deterministic scorer refuses nothing, so the loop labels (the round + /// counts items of the answerer's family) and does not forbid. /// `family` is `Unrecorded` when nothing records one. `generator` is the /// string the fixture or run wrote, kept verbatim. ModelWritten { @@ -54,7 +57,7 @@ pub enum Source { } /// The family of the model that wrote a line. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum ModelFamily { Anthropic, OpenAi, @@ -63,6 +66,7 @@ pub enum ModelFamily { Meta, Mistral, /// Nothing records which family wrote the line. + #[default] Unrecorded, } diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 2efb331..a71a0b9 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -64,9 +64,12 @@ pub enum DraftClass { UnknownField, /// A value of the wrong JSON type, such as a bare integer for an id. InvalidType, - /// A value of the right type that is not one of the field's: a node id - /// spelled `entity:x`, a number out of range. + /// A value of the right type that is not one of the field's, such as a + /// number out of range. InvalidValue, + /// A string that is not the form its field names: a node id spelled + /// `entity:x`, an id of the wrong kind ([`crate::form::FormError`]). + BadForm, Other, } @@ -80,6 +83,7 @@ impl DraftClass { DraftClass::UnknownField => "unknown-field", DraftClass::InvalidType => "invalid-type", DraftClass::InvalidValue => "invalid-value", + DraftClass::BadForm => "bad-form", DraftClass::Other => "other", } } @@ -102,6 +106,8 @@ impl DraftClass { DraftClass::InvalidType } else if msg.starts_with("invalid value") { DraftClass::InvalidValue + } else if msg.starts_with("not a ") { + DraftClass::BadForm } else { DraftClass::Other } @@ -312,6 +318,31 @@ mod tests { CorpusItemId::parse("item:daodejing:1").unwrap() } + #[test] + fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() { + let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class; + assert_eq!(class("not json"), DraftClass::Syntax); + assert_eq!(class("{\"referents\": ["), DraftClass::Eof); + assert_eq!(class("{}"), DraftClass::MissingField); + // The classes serde's own messages open with, read off a plain error. + let of = |r: Result| DraftClass::of(&r.unwrap_err()); + assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue); + assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType); + // Every code is distinct and none can hold answer text: they are fixed. + let all = [ + DraftClass::Syntax, + DraftClass::Eof, + DraftClass::MissingField, + DraftClass::UnknownField, + DraftClass::InvalidType, + DraftClass::InvalidValue, + DraftClass::BadForm, + DraftClass::Other, + ]; + let codes: std::collections::BTreeSet<_> = all.iter().map(|c| c.code()).collect(); + assert_eq!(codes.len(), all.len()); + } + #[test] fn ids_are_pinned_digests_of_their_inputs() { let encoder = encoder_id("fake-model", "abcd"); diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 3808672..5b88c14 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -92,6 +92,10 @@ enum LoopCommand { /// rounds only) or tool-less-relay. Pinned in the run's record. #[arg(long)] lowerer_reach: LowererReach, + /// The family of the answering model; rounds count the corpus items + /// that family wrote. Defaults to unrecorded. + #[arg(long, default_value = "unrecorded")] + answerer_family: ModelFamilyArg, }, /// Run one round of the run at ROOT. The stores are read only from the /// run's record: the global --corpus, --inventory and --entities are not @@ -744,6 +748,7 @@ fn main() -> Result<()> { answerer, embed_model, lowerer_reach, + answerer_family, }) => { let run = RunStores::create( &root, @@ -756,6 +761,7 @@ fn main() -> Result<()> { &answerer, &embed_model, lowerer_reach, + answerer_family.into(), )?; println!("opened run:{}", run.record().label); } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 517101d..6d0195f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -35,7 +35,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; -use crate::corpus::{CorpusItem, CorpusItemId, Source}; +use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; @@ -107,6 +107,12 @@ pub struct RunRecord { /// How the run's lowering prompts are answered, set once at open. #[cultcache(key = 12)] pub lowerer_reach: LowererReach, + /// The family of the model answering the run's encode and lowering + /// prompts, set once at open; `Unrecorded` when the opener did not say + /// and for a run opened before this was kept. + #[cultcache(key = 13, default)] + #[schemars(default)] + pub answerer_family: ModelFamily, } /// How a run's lowering prompts reach the model that answers them (rulings @@ -224,6 +230,8 @@ pub struct RefusalCounts { /// inventory_digest: String::new(), /// cycle: None, /// admit_rules: vec![], +/// parse_reasons: vec![], +/// own_family: 0, /// } /// } /// ``` @@ -276,9 +284,22 @@ pub struct RoundRecord { cycle: Option, /// The rules admission refused items under, in rule order; every Train /// and Validation item refused at `Stage::Admit` is counted once per rule. - #[cultcache(key = 18)] + #[cultcache(key = 18, default)] #[schemars(default)] admit_rules: Vec, + /// Why items were refused at `Stage::Parse`, in code order, over Train and + /// Validation: a [`DraftClass`] code (`rule`), or `unreadable` for an + /// item with no answer that read as text. Never a token of an answer. + #[cultcache(key = 19, default)] + #[schemars(default)] + parse_reasons: Vec, + /// Offered (live Train and Validation) `ModelWritten` items whose writer + /// family is the run's answerer family. The corpus rule binds + /// model judges; the scorer is deterministic, so nothing is refused for + /// this: the round labels it. + #[cultcache(key = 20, default)] + #[schemars(default)] + own_family: u32, } impl RoundRecord { @@ -336,6 +357,12 @@ impl RoundRecord { pub fn admit_rules(&self) -> &[RuleCount] { &self.admit_rules } + pub fn parse_reasons(&self) -> &[RuleCount] { + &self.parse_reasons + } + pub fn own_family(&self) -> u32 { + self.own_family + } /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -369,16 +396,24 @@ impl RoundRecord { x.admit, x.lower, ); - if self.admit_rules.is_empty() { - line - } else { - let rules: Vec = self - .admit_rules + let codes = |counts: &[RuleCount]| { + counts .iter() .map(|r| format!("{}x{}", r.rule, r.count)) - .collect(); - format!("{line} admit-rules={}", rules.join(",")) + .collect::>() + .join(",") + }; + let mut line = line; + if !self.parse_reasons.is_empty() { + line = format!("{line} parse-reasons={}", codes(&self.parse_reasons)); } + if !self.admit_rules.is_empty() { + line = format!("{line} admit-rules={}", codes(&self.admit_rules)); + } + if self.own_family > 0 { + line = format!("{line} own-family {}", self.own_family); + } + line } } @@ -550,13 +585,15 @@ impl RunStores { model: &str, embedder: &str, reach: LowererReach, + answerer_family: ModelFamily, ) -> Result { - Self::create_between(root, label, paths, model, embedder, reach, || {}) + Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {}) } /// `create` with `between` run after the digests are taken and before the /// stores are opened: the seam that lets a test change a store in that window. #[doc(hidden)] + #[allow(clippy::too_many_arguments)] // `create`'s six plus the test seam pub fn create_between( root: &Path, label: &str, @@ -564,6 +601,7 @@ impl RunStores { model: &str, embedder: &str, reach: LowererReach, + answerer_family: ModelFamily, between: impl FnOnce(), ) -> Result { if !is_slug(label) { @@ -592,6 +630,7 @@ impl RunStores { chrf_version: CHRF_VERSION.to_string(), inventory_digest: String::new(), lowerer_reach: reach, + answerer_family, }; // The run works on its own copy; the caller's store is never opened again. let head = root.join(INVENTORY_FILE); @@ -1024,13 +1063,21 @@ fn score_round( let mut items: Vec = Vec::new(); let mut fates: Vec = Vec::new(); let mut admit_rules: BTreeMap = BTreeMap::new(); + let mut parse_reasons: BTreeMap = BTreeMap::new(); // Phase 1 ends at admission: each admitted view waits for its lowering. let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new(); for (key, answer) in &answers { let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?; let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?; let fate = match answer.as_deref().map(PacketDraft::parse) { - None | Some(Err(_)) => Fate::Refused(Stage::Parse, None), + None => { + *parse_reasons.entry("unreadable".to_string()).or_default() += 1; + Fate::Refused(Stage::Parse, None) + } + Some(Err(e)) => { + *parse_reasons.entry(e.class.code().to_string()).or_default() += 1; + Fate::Refused(Stage::Parse, None) + } Some(Ok(draft)) => { let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); @@ -1172,6 +1219,15 @@ fn score_round( } } + let own_family = match run.record.answerer_family { + ModelFamily::Unrecorded => 0, + family => items + .iter() + .filter(|item| { + matches!(&item.source, Source::ModelWritten { family: f, .. } if *f == family) + }) + .count() as u32, + }; let n = run.rounds().map_err(store)?.len() as u32 + 1; let record = RoundRecord { n, @@ -1195,6 +1251,11 @@ fn score_round( .into_iter() .map(|(rule, count)| RuleCount { rule, count }) .collect(), + parse_reasons: parse_reasons + .into_iter() + .map(|(rule, count)| RuleCount { rule, count }) + .collect(), + own_family, }; Ok(RoundOutcome::Recorded(record)) } @@ -1628,6 +1689,8 @@ mod tests { inventory_digest: String::new(), cycle: None, admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, }; assert_eq!(r.n(), 1); } @@ -1844,6 +1907,8 @@ mod tests { inventory_digest: digest.to_string(), cycle: None, admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, } } @@ -1861,6 +1926,7 @@ mod tests { chrf_version: String::new(), inventory_digest: digest.to_string(), lowerer_reach: LowererReach::ToolLessRelay, + answerer_family: ModelFamily::Unrecorded, } } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index be64338..56a32af 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1346,6 +1346,8 @@ fn loop_open_and_unanswered_round() { "fake-model", "--lowerer-reach", "tool-less-relay", + "--answerer-family", + "open-ai", ], ) }; @@ -1395,7 +1397,7 @@ fn loop_open_and_unanswered_round() { assert!(second.status.success(), "{}", out(&second)); assert_eq!( out(&second), - "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 lowering=tool-less-relay\n" + "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" ); // The global store flags are not read by loop round: a drifted store is refused. diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 189202b..7bd9338 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -131,7 +131,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { entities: &dir.join("entities.cc"), }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, ) .unwrap() } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 377e8f3..60abc38 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use common::{ent, run_over, world}; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; +use weksa::corpus::ModelFamily; use weksa::run::{DriftOf, RunError, RunPaths, RunRecord, RunStores}; use weksa::split::fnv1a64; @@ -2947,6 +2948,282 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { assert!(!dao.run.round_line(&r).contains("admit-rules")); } +/// Item 1's good draft as JSON, changed by `change`, as the model's answer. +fn mangled(change: impl FnOnce(&mut serde_json::Value)) -> String { + let mut v: serde_json::Value = serde_json::from_str(&json(&draft_of(1))).unwrap(); + change(&mut v); + v.to_string() +} + +/// The round record of a Dao run whose item 1 is answered with `answer` and +/// the rest well. +fn round_with_item_1(answer: String) -> (Dao, RoundRecord) { + let mut dao = dao(); + let mut answers = good_answers(); + answers[0].1 = answer; + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + (dao, r) +} + +fn reasons(r: &RoundRecord) -> Vec<(&str, u32)> { + r.parse_reasons() + .iter() + .map(|c| (c.rule.as_str(), c.count)) + .collect() +} + +#[test] +fn parse_reasons_by_class() { + let good = json(&draft_of(1)); + let truncated = good[..good.len() / 2].to_string(); + let cases: Vec<(&str, String)> = vec![ + ("syntax", "not json at all".to_string()), + ("eof", truncated), + ( + "invalid-type", + mangled(|v| v["referents"][0]["concept"] = serde_json::json!(7)), + ), + ( + "bad-form", + mangled(|v| v["referents"][0]["concept"] = serde_json::json!("entity:x")), + ), + ( + "unknown-field", + mangled(|v| v["surprise"] = serde_json::json!(1)), + ), + ( + "missing-field", + mangled(|v| { + v.as_object_mut().unwrap().remove("referents"); + }), + ), + ]; + for (code, answer) in cases { + let (dao, r) = round_with_item_1(answer); + assert_eq!(reasons(&r), [(code, 1)], "{code}"); + assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}"); + let line = dao.run.round_line(&r); + assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}"); + } + // A round nothing refused at Parse carries and prints none. + let (dao, r) = round_with_item_1(good); + assert!(r.parse_reasons().is_empty()); + assert!(!dao.run.round_line(&r).contains("parse-reasons")); +} + +#[test] +fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { + let mut dao = dao(); + let expected = prompts(&dao.run); + let answers = keyed(&good_answers()); + let round1 = dao.root.join("round-1"); + std::fs::create_dir_all(&round1).unwrap(); + for (key, p) in &expected { + let bytes = match key.as_str() { + // Train: not UTF-8, so no answer parsed from the file. + "item:daodejing:1" => vec![0xff, 0xfe], + // Validation: not JSON. + "item:daodejing:2" => b"nope".to_vec(), + _ => answers[key].clone().into_bytes(), + }; + std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap(); + } + let table = table_english(); + for (key, p) in lowering_prompts(&dao, dao.run.inventory()) + .iter() + .filter(|(key, _)| key == "item:daodejing:3") + { + let answer = serde_json::json!({ "text": table[key] }).to_string(); + std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); + } + let mut files = AnswerFiles::new(round1, "fake-model".into()); + let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); + assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); +} + +#[test] +fn parse_reasons_never_echo_answer() { + let canary = "CANARY-7f3a-held-out"; + // The canary sits where serde would quote it: an unknown field name, an + // invalid value, and the raw text of a non-JSON answer. + let answers = [ + mangled(|v| v[canary] = serde_json::json!(1)), + mangled(|v| v["referents"][0]["concept"] = serde_json::json!(canary)), + format!("{canary} is not json"), + ]; + for answer in answers { + // Item 2 is Validation: the held-out item carries the canary. + let mut dao = dao(); + let mut a = good_answers(); + a[1].1 = answer; + let mut model = fake(&a); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(r.validation_refused().parse, 1); + let line = dao.run.round_line(&r); + assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}"); + let bytes = std::fs::read(dao.root.join("run.cc")).unwrap(); + assert!(!bytes.windows(6).any(|w| w == b"CANARY")); + assert!(!format!("{r:?}").contains("CANARY")); + } +} + +/// A run record as written before `answerer_family` (key 13) existed. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_run")] +struct PreFamilyRun { + #[cultcache(key = 0)] + label: String, + #[cultcache(key = 1)] + corpus: String, + #[cultcache(key = 2)] + inventory: String, + #[cultcache(key = 3)] + entities: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + prompt_digest: String, + #[cultcache(key = 8)] + model: String, + #[cultcache(key = 9)] + embedder: String, + #[cultcache(key = 10)] + chrf_version: String, + #[cultcache(key = 11)] + inventory_digest: String, + #[cultcache(key = 12)] + lowerer_reach: weksa::run::LowererReach, +} + +/// A round record as written before `parse_reasons` and `own_family`, and +/// before `admit_rules` had a default. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct LegacyRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 11)] + train_mean: weksa::score::SplitMean, + #[cultcache(key = 12)] + validation_mean: weksa::score::SplitMean, + #[cultcache(key = 13)] + validation_refused: weksa::run::RefusalCounts, + #[cultcache(key = 14)] + train_scored: u32, + #[cultcache(key = 15)] + validation_scored: u32, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, +} + +#[test] +fn records_from_before_the_new_fields_decode_with_defaults() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rec = dao.run.record().clone(); + let root = dao.root.clone(); + let legacy = LegacyRound { + n: r.n(), + inventory_rev: r.inventory_rev(), + prompt_digest: r.prompt_digest().into(), + corpus_digest: r.corpus_digest().into(), + entity_digest: r.entity_digest().into(), + model: r.model().into(), + embedder: r.embedder().into(), + chrf_version: r.chrf_version().into(), + calls: r.calls().to_vec(), + train: r.train().to_vec(), + train_mean: r.train_mean().clone(), + validation_mean: r.validation_mean().clone(), + validation_refused: r.validation_refused(), + train_scored: r.train_scored(), + validation_scored: r.validation_scored(), + inventory_digest: r.inventory_digest().into(), + cycle: r.cycle(), + }; + let run = PreFamilyRun { + label: rec.label.clone(), + corpus: rec.corpus.clone(), + inventory: rec.inventory.clone(), + entities: rec.entities.clone(), + corpus_digest: rec.corpus_digest.clone(), + entity_digest: rec.entity_digest.clone(), + prompt_digest: rec.prompt_digest.clone(), + model: rec.model.clone(), + embedder: rec.embedder.clone(), + chrf_version: rec.chrf_version.clone(), + inventory_digest: rec.inventory_digest.clone(), + lowerer_reach: rec.lowerer_reach, + }; + drop(dao.run); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &run).unwrap(); + cache.put("run:dao:round:1".to_string(), &legacy).unwrap(); + drop(cache); + + let reopened = RunStores::open(&root).unwrap(); + assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded); + let rounds = reopened.rounds().unwrap(); + assert_eq!(rounds.len(), 1); + assert_eq!(rounds[0].n(), 1); + assert!(rounds[0].admit_rules().is_empty()); + assert!(rounds[0].parse_reasons().is_empty()); + assert_eq!(rounds[0].own_family(), 0); +} + +#[test] +fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { + let published = weksa::schema::published(); + let schema = |needle: &str| -> serde_json::Value { + let (_, json) = published + .iter() + .find(|(name, _)| name.contains(needle)) + .unwrap(); + serde_json::from_str(json).unwrap() + }; + for (needle, fields) in [ + ("loop_round", &["parse_reasons", "own_family"][..]), + ("loop_run", &["answerer_family"][..]), + ] { + let s = schema(needle); + let required = s["required"].as_array().unwrap(); + for f in fields { + assert!(s["properties"][f].is_object(), "{needle} {f}"); + assert!(!required.iter().any(|r| r == f), "{needle} {f} is required"); + } + } +} + /// Answers a recall probe from the corpus: the item's own second half, a /// fixed wrong text, or junk. struct Reciter { diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index bbf7056..cffd4f2 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -112,3 +112,79 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { assert!(model.0.iter().all(|t| t.contains("work: not published"))); assert!(model.0.iter().all(|t| !t.contains("translator:"))); } + +/// Answers every prompt with text that is not a draft: nothing is admitted, so +/// the round measures only what is counted before admission. +struct Nope; + +impl weksa::model::ModelPort for Nope { + fn model(&self) -> &str { + "nope" + } + fn answer( + &mut self, + _: &weksa::context::Prompt, + ) -> Result { + Ok("not a draft".to_string()) + } +} + +struct Bag; + +impl weksa::score::Embedder for Bag { + fn model(&self) -> &str { + "bag" + } + fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts.iter().map(|_| vec![1.0]).collect()) + } +} + +fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::RoundRecord) { + use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap().keep(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(src.join(f), tmp.join(f)).unwrap(); + } + let mut run = RunStores::create( + &tmp.join("run"), + "orig", + RunPaths { + corpus: &tmp.join("corpus.cc"), + inventory: &tmp.join("inventory.cc"), + entities: &tmp.join("entity.cc"), + }, + "nope", + "bag", + LowererReach::ToolLessRelay, + family, + ) + .unwrap(); + let scorer = weksa::score::Scorer::new(Bag); + match round(&mut run, &mut Nope, &scorer).unwrap() { + RoundOutcome::Recorded(r) => (run, r), + RoundOutcome::Unanswered { .. } => panic!("answered"), + } +} + +#[test] +fn own_family_counts_and_labels() { + // The original corpus is six Anthropic-written items, all offered. + let (run, r) = original_round(ModelFamily::Anthropic); + assert_eq!(r.own_family(), 6); + assert_eq!( + r.parse_reasons().iter().map(|c| c.count).sum::(), + 6, + "every item was refused at Parse and says why" + ); + let line = run.round_line(&r); + assert!(line.contains(" own-family 6"), "{line}"); + + // A different answerer family, or none recorded, labels nothing. + for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { + let (run, r) = original_round(family); + assert_eq!(r.own_family(), 0); + assert!(!run.round_line(&r).contains("own-family")); + } +} diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index c82c4da..b5e5bc9 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -218,7 +218,7 @@ ] }, { - "description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.", + "description": "Labelled model output; never in the test split. The rule that it is\nnever judged by its own family binds model judges and the split; a\ndeterministic scorer refuses nothing, so the loop labels (the round\ncounts items of the answerer's family) and does not forbid.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.", "type": "object", "properties": { "ModelWritten": { diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index f218bdb..4117573 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n }\n}\n```", "type": "object", "properties": { "admit_rules": { @@ -57,6 +57,21 @@ "format": "uint32", "minimum": 0 }, + "own_family": { + "description": "Offered (live Train and Validation) `ModelWritten` items whose writer\nfamily is the run's answerer family. The corpus rule binds\nmodel judges; the scorer is deterministic, so nothing is refused for\nthis: the round labels it.", + "type": "integer", + "format": "uint32", + "default": 0, + "minimum": 0 + }, + "parse_reasons": { + "description": "Why items were refused at `Stage::Parse`, in code order, over Train and\nValidation: a [`DraftClass`] code (`rule`), or `unreadable` for an\nitem with no answer that read as text. Never a token of an answer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/RuleCount" + } + }, "prompt_digest": { "type": "string" }, diff --git a/schemas/cultnet/weksa.loop_run.schema.json b/schemas/cultnet/weksa.loop_run.schema.json index b5b5ab8..b339def 100644 --- a/schemas/cultnet/weksa.loop_run.schema.json +++ b/schemas/cultnet/weksa.loop_run.schema.json @@ -4,6 +4,11 @@ "description": "What a run pinned at open. Key `run: