From 17f44144a5372ec80eefd318c31a8def35986b27 Mon Sep 17 00:00:00 2001 From: Weksa Date: Sun, 8 Nov 2026 00:50:39 +0000 Subject: [PATCH 01/93] Show her working: static page of the lowering chain from the real lowerer and scorer Co-Authored-By: Claude Sonnet 5.5 --- .gitignore | 1 + crates/weksa/examples/show_working.rs | 373 ++++++++++++++++++++++++++ scripts/publish-show-working.sh | 16 ++ tools/show-working/README.md | 9 + tools/show-working/build.py | 203 ++++++++++++++ 5 files changed, 602 insertions(+) create mode 100644 crates/weksa/examples/show_working.rs create mode 100755 scripts/publish-show-working.sh create mode 100644 tools/show-working/README.md create mode 100755 tools/show-working/build.py diff --git a/.gitignore b/.gitignore index 1428a57..e00572b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ build/ .venv/ venv/ target/ +tools/show-working/out/ diff --git a/crates/weksa/examples/show_working.rs b/crates/weksa/examples/show_working.rs new file mode 100644 index 0000000..cf24b64 --- /dev/null +++ b/crates/weksa/examples/show_working.rs @@ -0,0 +1,373 @@ +//! Build-time generator for the "Show her working" page. Runs the real +//! admission door, lowerer and IPA reader over the committed proof packet and +//! variants of it, and prints one JSON document on stdout: the typed slots, the +//! inventory entries the packet chose, and the outcome, which is lowered text or +//! a named refusal. Scoring is not here: `tools/show-working/build.py` calls the +//! `weksa score` CLI on the text this emits. +//! +//! Run from the repository root: `cargo run --release --example show_working`. +//! Every variant is a mutation of `proof_packet()`, spelled in its `source`. + +use std::path::Path; + +use serde_json::{Value, json}; +use weksa::packet::{ + EntityId, FeatValue, FeatureValue, Grade, NodeId, ProvenanceSource, RoleFill, proof_packet, +}; +use weksa::{ + CorpusStore, EntityStore, InvId, Inventory, InventoryStore, LowerError, Packet, PacketStore, + StoreError, lower, pronounce, +}; + +struct Fixture { + id: &'static str, + title: &'static str, + note: &'static str, + source: &'static str, + packet: Packet, +} + +fn id(s: &str) -> InvId { + s.parse().expect("inventory id") +} + +fn node(s: &str) -> NodeId { + NodeId::parse(s).expect("node id") +} + +fn grade(feature: &str, v: u16) -> FeatureValue { + FeatureValue { + feature: id(feature), + value: FeatValue::Grade(Grade::new(v).expect("grade")), + } +} + +fn fixtures() -> Vec { + let proof = proof_packet(); + let mut out = vec![Fixture { + id: "proof", + title: "Nibu warns the player off the control", + note: "The committed proof packet, unchanged.", + source: "proof_packet() in crates/weksa/src/packet.rs", + packet: proof.clone(), + }]; + let mut add = |id, title, note, source, f: &dyn Fn(&mut Packet)| { + let mut packet = proof.clone(); + f(&mut packet); + out.push(Fixture { id, title, note, source, packet }); + }; + add( + "no-addressee", + "Same meaning, nobody addressed", + "With the addressee unset the player is a third party: no \"you\", no reflexive.", + "proof_packet() with discourse.addressee = None", + &|p| p.discourse.addressee = None, + ); + add( + "stance-held-back", + "Anger held back, restraint high", + "Stance is the speaker's felt affect. The lowerer has no row that reads it, so the text is the proof's text.", + "proof_packet() with stance anger=300, restraint=700", + &|p| p.stance.features = vec![grade("feature:9", 300), grade("feature:18", 700)], + ); + add( + "hedged", + "Hedged and uncertain warning", + "Hedge and certainty are meaning the lowerer cannot yet say. It refuses rather than drop them.", + "proof_packet() with discourse hedge=600, certainty=400", + &|p| p.discourse.features = vec![grade("feature:6", 600), grade("feature:7", 400)], + ); + add( + "negated", + "Negated intervention (polarity: negative)", + "Dropping the negation would say the opposite, so the lowerer refuses. This is the shape of a negated Dao line awaiting polarity.", + "proof_packet() with predication n5 polarity=negative", + &|p| { + p.predications[1].features = vec![FeatureValue { + feature: id("feature:1"), + value: FeatValue::Option("negative".parse().expect("label")), + }] + }, + ); + add( + "consent-authored", + "Addressee consent, authored source", + "Someone who saw the request says the addressee asked for it. No lowering speaks for that request yet.", + "proof_packet() with discourse.addressee_consent = n4 (source Authored)", + &|p| p.discourse.addressee_consent = Some(node("n4")), + ); + add( + "consent-imported", + "Addressee consent, imported-corpus source", + "A reader of wording cannot have seen a request, so the packet is refused at admission (V15), before any lowering.", + "proof_packet() with addressee_consent = n4 and provenance.source = ImportedCorpus", + &|p| { + p.discourse.addressee_consent = Some(node("n4")); + p.provenance.source = ProvenanceSource::ImportedCorpus; + }, + ); + add( + "relation", + "A causal relation between the predications", + "Relations are not rendered; the lowerer refuses instead of asserting a different meaning.", + "proof_packet() with relation:8 (causes) n4 -> n5", + &|p| { + p.relations = vec![weksa::packet::Relation { + relation: id("relation:8"), + from: node("n4"), + to: node("n5"), + features: vec![], + }] + }, + ); + add( + "dangling-filler", + "A role filled by a node that does not exist", + "Refused at admission (V4): the packet is not a packet.", + "proof_packet() with predication n5 agent filler n9", + &|p| { + p.predications[1].roles[0] = RoleFill { role: id("role:1"), filler: node("n9") } + }, + ); + add( + "unknown-entity", + "A referent naming an entity nobody holds", + "The entity store is the only owner of names; a packet cannot invent one.", + "proof_packet() with referent n3 entity = entity:nobody", + &|p| p.referents[2].entity = Some(EntityId::parse("entity:nobody").expect("entity")), + ); + add( + "common-noun-agent", + "The ship mind as a common noun", + "No entity: the referent lowers as \"the ship mind\", and there is no name to pronounce.", + "proof_packet() with referent n3 entity = None", + &|p| p.referents[2].entity = None, + ); + out +} + +fn gloss(inv: &Inventory, rev: u32, i: &InvId) -> Value { + match inv.resolve(i, rev) { + Some(e) => json!({ "id": i.to_string(), "gloss": e.gloss }), + None => json!({ "id": i.to_string(), "gloss": null }), + } +} + +fn feature_json(inv: &Inventory, rev: u32, slot: &str, at: &str, f: &FeatureValue) -> Value { + let mut g = gloss(inv, rev, &f.feature); + g["slot"] = json!(slot); + g["at"] = json!(at); + g["value"] = serde_json::to_value(&f.value).expect("value"); + g +} + +/// Every feature the packet carries, with where it sits. +fn features(p: &Packet, inv: &Inventory) -> Vec { + let rev = p.inventory_rev; + let mut v = Vec::new(); + for f in &p.discourse.features { + v.push(feature_json(inv, rev, "discourse", "discourse", f)); + } + for f in &p.stance.features { + v.push(feature_json(inv, rev, "stance", "stance", f)); + } + for r in &p.referents { + for f in &r.features { + v.push(feature_json(inv, rev, "referent", r.id.as_str(), f)); + } + } + for pr in &p.predications { + for f in &pr.features { + v.push(feature_json(inv, rev, "predication", pr.id.as_str(), f)); + } + } + v +} + +/// Ids the packet chose, once each, with the inventory's gloss at its revision. +fn chosen(p: &Packet, inv: &Inventory) -> Vec { + let rev = p.inventory_rev; + let mut ids: Vec = Vec::new(); + let mut push = |i: &InvId| { + if !ids.contains(i) { + ids.push(*i) + } + }; + p.context.setting.iter().for_each(&mut push); + p.context.medium.iter().for_each(&mut push); + p.discourse.speech_act.iter().for_each(&mut push); + for r in &p.referents { + push(&r.kind); + push(&r.concept); + } + for pr in &p.predications { + push(&pr.concept); + push(&pr.frame); + pr.roles.iter().for_each(|f| push(&f.role)); + } + p.relations.iter().for_each(|r| push(&r.relation)); + features(p, inv) + .iter() + .filter_map(|f| f["id"].as_str().and_then(|s| s.parse().ok())) + .for_each(|i: InvId| push(&i)); + ids.iter().map(|i| gloss(inv, rev, i)).collect() +} + +fn slots(p: &Packet) -> Value { + let j = |v: &dyn erased::Ser| v.to_value(); + json!({ + "meaning": { + "referents": p.referents.iter().map(|r| j(r)).collect::>(), + "predications": p.predications.iter().map(|r| j(r)).collect::>(), + "relations": p.relations.iter().map(|r| j(r)).collect::>(), + }, + "stance": j(&p.stance), + "discourse": j(&p.discourse), + "context": j(&p.context), + "extensions": p.extensions.iter().map(|r| j(r)).collect::>(), + "constraints": j(&p.constraints), + "provenance": { "source": j(&p.provenance.source), "corpus_item": p.provenance.corpus_item.to_string(), "confidence": p.provenance.confidence.get() }, + "kind": j(&p.kind), + "inventory_rev": p.inventory_rev, + }) +} + +mod erased { + pub trait Ser { + fn to_value(&self) -> serde_json::Value; + } + impl Ser for T { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(self).expect("serialize") + } + } +} + +fn lower_error(e: &LowerError) -> Value { + let (variant, reason) = match e { + LowerError::Unrendered { what } => ("Unrendered", format!("the packet carries {what}, which the lowerer does not render")), + LowerError::NoRow { id } => ("NoRow", format!("no table row for {id}")), + LowerError::GlossMismatch { id, row } => ("GlossMismatch", format!("{id} is not the {row} the row speaks for")), + LowerError::UnknownRole { frame, role } => ("UnknownRole", format!("{frame} names role {role}, which the inventory lacks")), + LowerError::MissingRole { predication, role } => ("MissingRole", format!("{predication} fills no {role} role")), + LowerError::UnknownEntity { entity } => ("UnknownEntity", format!("the entity store does not hold {entity}")), + LowerError::NoForm { entity } => ("NoForm", format!("{entity} has no en-US form")), + LowerError::NoPronunciation { entity } => ("NoPronunciation", format!("{entity} has no ruled en-US pronunciation")), + LowerError::NoEntityReferent => ("NoEntityReferent", "the packet cites no entity".to_string()), + LowerError::EntityRead { entity, why } => ("EntityRead", format!("reading {entity}: {why}")), + }; + json!({ "variant": variant, "reason": reason, "debug": format!("{e:?}") }) +} + +struct World { + inventory: InventoryStore, + corpus: CorpusStore, + entities: EntityStore, + inv: Inventory, +} + +/// Admit `packet` through a fresh packet store and lower it. `Ok` carries the +/// text and the IPA outcome; `Err` is the refusal, as JSON. +fn run(w: &World, dir: &Path, packet: &Packet) -> Result<(String, Value), Value> { + let mut store = PacketStore::open(&dir.join("p.cc")).expect("packet store"); + let view = match store.admit(packet.clone(), &w.inventory, &w.corpus, &w.entities) { + Ok(v) => v, + Err(StoreError::Invalid(rs)) => { + return Err(json!({ + "kind": "admission_refused", + "refusals": rs.iter().map(|r| json!({ "rule": r.rule(), "detail": format!("{r:?}") })).collect::>(), + })); + } + Err(e) => { + return Err(json!({ "kind": "admission_refused", "refusals": [{ "rule": "store", "detail": format!("{e:?}") }] })); + } + }; + let text = lower(&view, &w.inv, &w.entities).map_err(|e| { + json!({ "kind": "lower_error", "error": lower_error(&e) }) + })?; + let ipa = match pronounce(&view, &w.entities) { + Ok(v) => json!({ "ok": v }), + Err(e) => json!({ "error": lower_error(&e) }), + }; + Ok((text, ipa)) +} + +fn main() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inventory = InventoryStore::open(&root.join("data/interlingua/inventory.cc")).expect("inventory"); + let inv = inventory.inventory().expect("inventory view"); + let w = World { + corpus: CorpusStore::open(&root.join("data/corpus/corpus.cc")).expect("corpus"), + entities: EntityStore::open(&root.join("data/entity/entity.cc")).expect("entities"), + inventory, + inv, + }; + let dir = std::env::temp_dir().join(format!("show-working-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("scratch dir"); + let mut out = Vec::new(); + for fx in fixtures() { + let p = &fx.packet; + let first = run(&w, &dir, p); + let outcome = match first { + Err(e) if e["kind"] == "admission_refused" => e, + first => { + // Ablation: run again with each feature removed, against the + // baseline outcome. It is how the page says whether the lowerer + // read a feature, instead of the page asserting it. + let base = first.as_ref().ok().map(|(t, _)| t.clone()); + let n_disc = p.discourse.features.len(); + let n_stance = p.stance.features.len(); + let total = features(p, &w.inv).len(); + let mut effects = Vec::new(); + for k in 0..total { + let mut q = p.clone(); + if k < n_disc { + q.discourse.features.remove(k); + } else if k < n_disc + n_stance { + q.stance.features.remove(k - n_disc); + } else { + let mut at = k - n_disc - n_stance; + 'o: { + for r in &mut q.referents { + if at < r.features.len() { r.features.remove(at); break 'o; } + at -= r.features.len(); + } + for pr in &mut q.predications { + if at < pr.features.len() { pr.features.remove(at); break 'o; } + at -= pr.features.len(); + } + } + } + let _ = std::fs::remove_file(dir.join("p.cc")); + effects.push(match (&base, run(&w, &dir, &q)) { + (Some(t), Ok((u, _))) if *t == u => "Without it the text is identical: the lowerer did not read it.", + (Some(_), Ok(_)) => "Without it the text changes: the lowerer read it.", + (Some(_), Err(_)) => "Without it the packet is refused.", + (None, Ok(_)) => "Without it the packet lowers: this feature is what the lowerer refuses.", + (None, Err(_)) => "Still refused without this feature alone.", + }); + } + match first { + Ok((text, ipa)) => json!({ "kind": "lowered", "text": text, "ipa": ipa, "feature_effects": effects }), + Err(e) => { let mut e = e; e["feature_effects"] = json!(effects); e } + } + } + }; + let _ = std::fs::remove_file(dir.join("p.cc")); + let mut slots = slots(p); + slots["features"] = json!(features(p, &w.inv)); + out.push(json!({ + "id": fx.id, + "title": fx.title, + "note": fx.note, + "source": fx.source, + "packet_id": p.id.to_string(), + "corpus_item": p.provenance.corpus_item.to_string(), + "slots": slots, + "chosen": chosen(p, &w.inv), + "outcome": outcome, + })); + } + let _ = std::fs::remove_dir_all(&dir); + println!("{}", serde_json::to_string_pretty(&json!({ "fixtures": out })).expect("json")); +} diff --git a/scripts/publish-show-working.sh b/scripts/publish-show-working.sh new file mode 100755 index 0000000..3af3194 --- /dev/null +++ b/scripts/publish-show-working.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Build the "Show her working" page from the real lowerer and scorer, then publish it as a Mind preview. +# Run from the repository root, in a Mind's Self run (it writes only /srv/minds-web//). +# +# scripts/publish-show-working.sh [--no-publish] +# +# Needs cargo, python3 and an Ollama embedder at 127.0.0.1:11434 (qwen3-embedding:0.6b): the scorer is not stubbed. +# publish-mind-preview.sh lives in the gamecult-ops checkout: set GAMECULT_OPS to it (default ../gamecult-ops). +set -euo pipefail +[ -f Cargo.toml ] && [ -d tools/show-working ] || { echo "run from the Weksa repository root" >&2; exit 2; } +out=${SHOW_WORKING_OUT:-tools/show-working/out} +rm -rf "$out" +python3 -I tools/show-working/build.py "$out" +[ "${1:-}" = --no-publish ] && { echo "built $out"; exit 0; } +ops=${GAMECULT_OPS:-../gamecult-ops} +exec "$ops/scripts/publish-mind-preview.sh" "$out" weksa-show-working diff --git a/tools/show-working/README.md b/tools/show-working/README.md new file mode 100644 index 0000000..5e2a320 --- /dev/null +++ b/tools/show-working/README.md @@ -0,0 +1,9 @@ +# Show her working + +A static page that shows the lowering chain for committed fixture packets: typed slots, inventory entries chosen, lowered English, scorer closeness to the corpus target, with hedge, certainty and restraint annotated. Refusals (admission rules, `LowerError`, addressee consent, negation awaiting polarity) are results, not omissions. + +- `crates/weksa/examples/show_working.rs`: runs the real admission door, `lower` and `pronounce` over `proof_packet()` and its variants and prints JSON. Each feature is removed in turn and the packet re-run, so "the lowerer read this" is measured. +- `build.py`: runs that, scores each lowered text with the `weksa score` CLI (needs Ollama with `qwen3-embedding:0.6b`), writes `out/data.json` and one dependency-free `out/index.html`. Fails if anything cannot run; there is no fallback text. +- `scripts/publish-show-working.sh`: build, then publish through gamecult-ops `publish-mind-preview.sh`. + +Adding a fixture: add an entry in `fixtures()` in the example. A packet from another store needs a loader there; none exists yet. diff --git a/tools/show-working/build.py b/tools/show-working/build.py new file mode 100755 index 0000000..269032e --- /dev/null +++ b/tools/show-working/build.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Build the "Show her working" page from the real lowerer and scorer. + +usage: build.py [out_dir] (run from the repository root) + +1. `cargo run --example show_working` admits each fixture packet through the + packet store and lowers it with the English lowerer (crates/weksa/examples). +2. `weksa score` scores each lowered text against its corpus item (and the + other committed Nibu reference) through the Ollama embedder. +3. out_dir/data.json holds the joined result; out_dir/index.html is one file + with no scripts to fetch and no network calls. + +Nothing on the page is typed by hand except the layout and the reading notes. +If cargo or the scorer cannot run, the build fails: there is no fallback text. +Standard library only. +""" +import json +import pathlib +import re +import subprocess +import sys +import time +from html import escape as e + +root = pathlib.Path.cwd() +out_dir = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else "tools/show-working/out") +weksa_bin = None + + +def run(cmd, **kw): + r = subprocess.run(cmd, cwd=root, capture_output=True, text=True, **kw) + if r.returncode != 0: + sys.exit(f"{' '.join(map(str, cmd))} failed:\n{r.stderr}") + return r.stdout + + +def cargo_target(): + meta = json.loads(run(["cargo", "metadata", "--format-version", "1", "--no-deps"])) + return pathlib.Path(meta["target_directory"]) + + +# -- 1. the real lowerer ------------------------------------------------------ +run(["cargo", "build", "--release", "--bin", "weksa", "--example", "show_working"]) +target = cargo_target() / "release" +weksa_bin = target / "weksa" +data = json.loads(run([str(target / "examples" / "show_working")])) + +# -- 2. the real scorer ------------------------------------------------------- +corpus = {} +for m in re.finditer(r'^(item:\S+) \[(.*?)\]\n "(.*)"$', run([str(weksa_bin), "corpus", "dump"]), re.M): + corpus[m.group(1)] = {"text": m.group(3), "meta": m.group(2)} +REFS_EXTRA = ["item:nibu:2"] +model = "qwen3-embedding:0.6b" + + +def score(text, items): + out = run([str(weksa_bin), "score", *items], input=text) + got = {} + for m in re.finditer(r"^(item:\S+) chrf=([\d.]+) meaning=(-?[\d.]+)", out, re.M): + got[m.group(1)] = {"chrf": float(m.group(2)), "meaning": float(m.group(3))} + if set(got) != set(items): + sys.exit(f"scorer answered {sorted(got)} for {items}:\n{out}") + return got + + +for fx in data["fixtures"]: + o = fx["outcome"] + if o["kind"] == "lowered": + refs = [fx["corpus_item"]] + [r for r in REFS_EXTRA if r != fx["corpus_item"]] + o["scores"] = [{"item": k, "target": k == fx["corpus_item"], **v} for k, v in score(o["text"], refs).items()] + +data["corpus"] = corpus +data["build"] = { + "commit": run(["git", "rev-parse", "HEAD"]).strip(), + "embedder": model, + "built_unix": int(time.time()), +} +out_dir.mkdir(parents=True, exist_ok=True) +(out_dir / "data.json").write_text(json.dumps(data, indent=1, ensure_ascii=False), encoding="utf-8") + +# -- 3. the page -------------------------------------------------------------- +CSS = """ +:root{--ink:#1c1b19;--mute:#5d5a54;--line:#cfc9bd;--bg:#f6f3ec;--card:#fff;--ok:#0b6b4b;--warn:#9a4a00;--bad:#9b1c2c;--bar:#365f9c} +*{box-sizing:border-box}body{font:16px/1.55 system-ui,sans-serif;max-width:62rem;margin:2rem auto;padding:0 1rem;color:var(--ink);background:var(--bg)} +h1{margin:0 0 .2rem}h2{margin:0 0 .4rem}h3{margin:1.2rem 0 .3rem;font-size:.85rem;text-transform:uppercase;letter-spacing:.06em;color:var(--mute)} +code,.mono{font:13px/1.4 ui-monospace,Menlo,monospace}code{background:#ece8dd;padding:0 .25rem;border-radius:3px} +.lede{max-width:46rem}.meta{color:var(--mute);font-size:.88rem} +nav.pick{display:flex;flex-wrap:wrap;gap:.4rem;margin:1.2rem 0} +nav.pick a{border:1px solid var(--line);background:var(--card);padding:.25rem .6rem;border-radius:999px;text-decoration:none;color:var(--ink);font-size:.9rem} +nav.pick a.refused{border-color:var(--bad)}nav.pick a.lowered{border-color:var(--ok)} +article{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:1rem 1.3rem;margin:1.4rem 0} +.chain{display:grid;grid-template-columns:repeat(4,1fr);gap:.6rem;margin:.6rem 0 1rem} +.chain div{border:1px solid var(--line);border-radius:6px;padding:.3rem .6rem;font-size:.82rem;color:var(--mute)} +.chain b{display:block;color:var(--ink)} +table{border-collapse:collapse;width:100%;font-size:.9rem}th,td{text-align:left;padding:.25rem .5rem;border-bottom:1px solid #e6e1d5;vertical-align:top}th{color:var(--mute);font-weight:600} +.out{font-size:1.15rem;padding:.6rem .9rem;border-left:4px solid var(--ok);background:#eef6f1} +.refusal{padding:.6rem .9rem;border-left:4px solid var(--bad);background:#faeef0}.refusal b{color:var(--bad)} +.tag{display:inline-block;font-size:.75rem;padding:0 .4rem;border-radius:3px;border:1px solid var(--line);color:var(--mute)} +.tag.read{border-color:var(--ok);color:var(--ok)}.tag.unread{border-color:var(--warn);color:var(--warn)}.tag.refuse{border-color:var(--bad);color:var(--bad)} +.bar{height:.7rem;background:#e4dfd2;border-radius:3px;min-width:8rem;position:relative}.bar i{display:block;height:100%;background:var(--bar);border-radius:3px} +.ref{font-size:.88rem;color:var(--mute);margin:.2rem 0 .6rem}.ref q{color:var(--ink)} +details{margin:.5rem 0}summary{cursor:pointer;color:var(--mute);font-size:.88rem} +section.gaps li{margin:.3rem 0} +@media(max-width:700px){.chain{grid-template-columns:1fr 1fr}} +""" + +PICK_JS = """(function(){var a=[].slice.call(document.querySelectorAll('article[data-fx]'));function show(){var h=location.hash.slice(1);var any=a.some(function(x){return x.id===h});a.forEach(function(x){x.hidden=any&&x.id!==h})} +window.addEventListener('hashchange',show);show()})()""" + + +def g(chosen, i): + return next((c["gloss"] for c in chosen if c["id"] == i), "?") + + +def val(v): + (k, x), = v.items() + return f"{x}‰" if k == "Grade" else str(x) + + +def slots_html(fx): + s, ch = fx["slots"], fx["chosen"] + rows = [] + for r in s["meaning"]["referents"]: + ent = f" · entity {e(r['entity'])}" if r["entity"] else "" + rows.append(f"{e(r['id'])}referent{e(g(ch, r['kind']))} / {e(g(ch, r['concept']))}{ent}") + for p in s["meaning"]["predications"]: + roles = ", ".join(f"{e(g(ch, x['role']))}={e(x['filler'])}" for x in p["roles"]) + rows.append(f"{e(p['id'])}predication{e(g(ch, p['concept']))} in frame {e(g(ch, p['frame']))}: {roles}") + for r in s["meaning"]["relations"]: + rows.append(f"relation{e(g(ch, r['relation']))}: {e(r['from'])} → {e(r['to'])}") + meaning = "" + "".join(rows) + "
nodekindcontent
" + d = s["discourse"] + drows = [] + if d["speech_act"]: + drows.append(f"speech act{e(g(ch, d['speech_act']))}") + if d["addressee"]: + drows.append(f"addressee{e(d['addressee'])}") + if d["addressee_consent"]: + drows.append(f"addressee consentpredication {e(d['addressee_consent'])} (what the addressee asked for)") + drows.append(f"source{e(s['provenance']['source'])}, kind {e(s['kind'])}") + return (f"

Slot: meaning

{meaning}

Slot: discourse

{''.join(drows)}
" + + stance_register(fx)) + + +def stance_register(fx): + s = fx["slots"] + eff = fx["outcome"].get("feature_effects") or [] + rows = [] + for i, f in enumerate(s["features"]): + note = eff[i] if i < len(eff) else "" + cls = "read" if "read it" in note else "unread" if "did not read" in note else "refuse" if "refus" in note else "" + rows.append(f"{e(f['gloss'])} {e(f['id'])}{e(f['slot'])} @ {e(f['at'])}{e(val(f['value']))}{e(note)}") + feats = ("" + "".join(rows) + "
featureslotvaluemeasured effect on the lowerer
") if rows else "

No features: no hedge, certainty, polarity, irony or stance set.

" + return ("

Slot: stance, hedge, certainty, restraint (features)

" + feats + + "

Register: the packet has no register slot. Register and address belong to a target-language profile, not to the packet; this lowerer reads none.

") + + +def outcome_html(fx): + o = fx["outcome"] + if o["kind"] == "lowered": + ipa = o["ipa"] + ipa_s = ("IPA of cited entities: " + e(" ".join(ipa["ok"])) + "") if "ok" in ipa else f"IPA refused: {e(ipa['error']['reason'])} {e(ipa['error']['variant'])}" + sc = [] + for s in o["scores"]: + ref = corpus[s["item"]]["text"] + sc.append(f"
{'Target' if s['target'] else 'Other reference'} {e(s['item'])}: {e(ref)}
" + f"" + f"
chrF {s['chrf']:.1f}
meaning cosine {s['meaning']:.3f}
") + return (f"

Lowered English

{e(o['text'])}

{ipa_s}

" + f"

Closeness to the corpus ({e(data['build']['embedder'])})

{''.join(sc)}") + if o["kind"] == "lower_error": + er = o["error"] + return f"

Refused by the lowerer

LowerError::{e(er['variant'])}: {e(er['reason'])}
{e(er['debug'])}

No text, so no score: nothing was invented to score.

" + items = "".join(f"
  • {e(r['rule'])} {e(r['detail'])}
  • " for r in o["refusals"]) + return f"

    Refused at admission

      {items}

    The packet never became a lowering input, so the lowerer never saw it.

    " + + +def card(fx): + o = fx["outcome"] + ch = "".join(f"{e(c['id'])}{e(c['gloss'] or '(unresolved)')}" for c in fx["chosen"]) + steps = [("1 Typed slots", "meaning, stance, discourse"), ("2 Inventory entries", f"{len(fx['chosen'])} ids, rev {fx['slots']['inventory_rev']}"), + ("3 Lowered English", {"lowered": "text", "lower_error": "refused", "admission_refused": "never reached"}[o["kind"]]), + ("4 Closeness", "scored" if o["kind"] == "lowered" else "no text, no score")] + chain = "".join(f"
    {e(a)}{e(b)}
    " for a, b in steps) + return (f"

    {e(fx['title'])}

    {e(fx['note'])}

    " + f"

    Packet {e(fx['packet_id'])} for {e(fx['corpus_item'])}. Made from: {e(fx['source'])}.

    " + f"
    {chain}
    {slots_html(fx)}" + f"
    Inventory entries chosen ({len(fx['chosen'])}){ch}
    idgloss at this revision
    " + f"{outcome_html(fx)}
    ") + + +nav = "".join(f"{e(f['title'])}" for f in data["fixtures"]) +b = data["build"] +page = f""" +Weksa: show her working +

    Show her working

    +

    A meaning packet goes in. The English lowerer picks inventory entries, writes a sentence, and a scorer measures how close that sentence is to a human-written target. Every number and sentence below was produced by running the real Rust lowerer and scorer when this page was built. The refusals are results too: when the lowerer cannot say a meaning without changing it, it says so instead of dropping the part it cannot say.

    +

    Built from {e(b['commit'][:10])}, embedder {e(b['embedder'])}. English only. Fixtures are the committed nibu-proof-001 packet and variants of it, each variant described in its own card. Scores are against the committed corpus: chrF (0-100) and embedding cosine. Raw data: data.json.

    + +{''.join(card(f) for f in data['fixtures'])} +""" +(out_dir / "index.html").write_text(page, encoding="utf-8") +print(f"wrote {out_dir}/index.html and data.json ({len(data['fixtures'])} fixtures)") From 00250088a8f572a3740c6f5a0916b5c54e38783d Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 13:58:28 +0000 Subject: [PATCH 02/93] Lows: summary_line leaves the public surface; lowering bound pinned in bytes, CR refused RoundRecord and Decision no longer expose an unlabelled summary_line; the only public spellings are RunStores::round_line and cycle_line, which label with the run's lowerer reach. The lowering bound tests now kill the chars-for-bytes and CR-allowed mutants. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 4 ++-- crates/weksa/src/main.rs | 4 ++-- crates/weksa/src/run.rs | 18 ++++++++++++++---- crates/weksa/tests/golf_loop.rs | 31 +++++++++++++++++++------------ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 9aa08a0..5071316 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -217,8 +217,8 @@ impl LoweringDraft { } /// Whether the text can be scored as an utterance: something after trimming, - /// at most [`MAX_LOWERING_BYTES`], and no control character other than a - /// line break or tab. Nothing is judged about the words. + /// at most [`MAX_LOWERING_BYTES`] bytes (not chars), and no control character other than a + /// newline or tab (a carriage return is refused). Nothing is judged about the words. pub fn usable(&self) -> bool { !self.text.trim().is_empty() && self.text.len() <= MAX_LOWERING_BYTES diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 148f0be..06b3cce 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -771,7 +771,7 @@ fn main() -> Result<()> { println!("unanswered {} of {asked}", prompts.len()); std::process::exit(3); } - RoundOutcome::Recorded(r) => println!("{}", run.labelled(&r.summary_line())), + RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)), } } Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { @@ -791,7 +791,7 @@ fn main() -> Result<()> { std::process::exit(3); } CycleOutcome::Decided(decision) => { - println!("{}", run.labelled(&decision.summary_line())) + println!("{}", run.cycle_line(&decision)) } } } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 1b6d6a3..3d41aa7 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -317,9 +317,9 @@ impl RoundRecord { self.cycle } - /// The one line a recorded round prints. `n` is every item of the split, + /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. - pub fn summary_line(&self) -> String { + fn line(&self) -> String { let mean = |value: Option, places: usize| { value.map_or("none".to_string(), |v| format!("{v:.places$}")) }; @@ -648,6 +648,16 @@ impl RunStores { } } + /// The line a recorded round prints, labelled with how the run was lowered. + pub fn round_line(&self, round: &RoundRecord) -> String { + self.labelled(&round.line()) + } + + /// The line a closed cycle prints, labelled the same way. + pub fn cycle_line(&self, decision: &Decision) -> String { + self.labelled(&decision.line()) + } + pub fn inventory(&self) -> &InventoryStore { &self.inventory } @@ -1224,8 +1234,8 @@ impl Decision { self.verdict } - /// The one line a closed cycle prints. - pub fn summary_line(&self) -> String { + /// The cycle line, unlabelled: [`RunStores::cycle_line`] adds how the run was lowered. + fn line(&self) -> String { let candidate_rev = self .candidate_rev .map_or("none".to_string(), |n| n.to_string()); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 58ee80d..a4bba52 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -1428,13 +1428,13 @@ fn the_round_line_prints_each_split_under_its_own_name() { let mut dao = dao(); let mut model = fake(&good_answers()); let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); - let line = r.summary_line(); + let line = dao.run.round_line(&r); // Train is item 1 alone, Validation items 2 and 3; the numbers differ. let (t, v) = (r.train_mean(), r.validation_mean()); assert_eq!( line, format!( - "round 1 rev 3 train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0", + "round 1 rev 3 train n=1 scored=1 chrf={:.2} meaning={:.4} refused=0 validation n=2 scored=2 chrf={:.2} meaning={:.4} refused parse=0 admit=0 lower=0 lowering=tool-less-relay", t.chrf.unwrap(), t.meaning.unwrap(), v.chrf.unwrap(), @@ -1534,11 +1534,11 @@ fn errors_echo_no_input() { } let d = decided(cycle(&mut canary_run, &mut model, &Scorer::new(Bag)).unwrap()); assert_eq!(d.verdict(), Verdict::Rejected(Why::EditParse)); - all.extend([format!("{d:?}"), d.summary_line()]); + all.extend([format!("{d:?}"), canary_run.cycle_line(&d)]); let mut non_utf8 = cycle_fake(""); non_utf8.edit = Some(Err(PortError::Io(std::io::ErrorKind::InvalidData))); let d = decided(cycle(&mut canary_run, &mut non_utf8, &Scorer::new(Bag)).unwrap()); - all.extend([format!("{d:?}"), d.summary_line()]); + all.extend([format!("{d:?}"), canary_run.cycle_line(&d)]); weksa::InventoryStore::open(&canary_root.join("inventory.cc")) .unwrap() .admit_revision( @@ -1726,9 +1726,9 @@ fn cycle_accepts_a_gain_past_the_floor() { // The line, with the readings printed to four places. let (b, c) = (d.base().composite, d.candidate().unwrap().composite); assert_eq!( - d.summary_line(), + dao.run.cycle_line(&d), format!( - "cycle 1 rev 3->4 base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted" + "cycle 1 rev 3->4 base={b:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 accepted lowering=tool-less-relay" ) ); @@ -1786,9 +1786,9 @@ fn cycle_rejects_no_gain_and_keeps_the_head() { assert_eq!(d.candidate(), Some(d.base())); let c = d.base().composite; assert_eq!( - d.summary_line(), + dao.run.cycle_line(&d), format!( - "cycle 1 rev 3->4 base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain" + "cycle 1 rev 3->4 base={c:.4} candidate={c:.4} floor=0.0000 threshold=0.0050 rejected no-gain lowering=tool-less-relay" ) ); assert_eq!(head_rev(&dao.root), 3); @@ -2207,9 +2207,9 @@ fn run_the_editor_canary(canary_lowered: &str) { // Nothing the cycle writes or prints holds a held-out key or text. for text in [ format!("{d:?}"), - d.summary_line(), + dao.run.cycle_line(&d), format!("{d2:?}"), - d2.summary_line(), + dao.run.cycle_line(&d2), ] { for needle in &held_out { assert!( @@ -2312,9 +2312,9 @@ fn edit_refusals_are_decisions() { "edit-admit" }; assert_eq!( - d.summary_line(), + dao.run.cycle_line(&d), format!( - "cycle 1 rev 3->none base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag}", + "cycle 1 rev 3->none base={:.4} candidate=none floor=0.0000 threshold=0.0050 rejected {tag} lowering=tool-less-relay", d.base().composite ), "{name}" @@ -2679,6 +2679,12 @@ fn lowering_refusals_count_at_lower() { ("non-UTF-8", Err(PortError::Io(std::io::ErrorKind::InvalidData))), ("NUL", Ok(serde_json::json!({ "text": "The sage\u{0}acts." }).to_string())), ("bell", Ok(serde_json::json!({ "text": "The sage \u{7}acts." }).to_string())), + ("carriage return", Ok(serde_json::json!({ "text": "The sage\racts." }).to_string())), + ( + "over length in bytes, within it in chars", + Ok(serde_json::json!({ "text": "\u{e9}".repeat(weksa::encode::MAX_LOWERING_BYTES / 2 + 1) }) + .to_string()), + ), ( "over length", Ok(serde_json::json!({ "text": "a".repeat(weksa::encode::MAX_LOWERING_BYTES + 1) }) @@ -2776,6 +2782,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { fn lowering_text_at_the_bound_is_scored_and_line_breaks_are_text() { for text in [ "a".repeat(weksa::encode::MAX_LOWERING_BYTES), + "\u{e9}".repeat(weksa::encode::MAX_LOWERING_BYTES / 2), "The sage acts.\n\tThe sage rests.".to_string(), ] { let mut dao = dao(); From c527d0eb084cf18c3f14f72f1c8620ecbecde3bb Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:05:13 +0000 Subject: [PATCH 03/93] RoundRecord names the admission rules that refused items; the round line prints them admit_rules (key 18) counts, over both splits, each rule an admission refusal broke: validator codes (V2, V5) or a store refusal's variant name. Rule codes only, no identifiers or text. The printed line gains admit-rules=... before the lowerer-reach label. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 68 ++++++++++++++++++-- crates/weksa/tests/golf_loop.rs | 38 +++++++++++ schemas/cultnet/weksa.loop_round.schema.json | 30 ++++++++- 3 files changed, 130 insertions(+), 6 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 3d41aa7..93ae120 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -23,6 +23,7 @@ //! uses the template table. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. +use std::collections::BTreeMap; use std::fmt; use std::io::ErrorKind; use std::path::{Path, PathBuf}; @@ -40,7 +41,9 @@ use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; use crate::score::{CHRF_VERSION, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; -use crate::store::{CorpusStore, EntityStore, InventoryStore, PacketStore, open_cache_over}; +use crate::store::{ + CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over, +}; use crate::validate::LoweringInput; /// The digest of the prompt builder's source: `context.rs`, then `split.rs`, @@ -164,6 +167,14 @@ pub struct Call { pub items: Vec, } +/// How many items one admission rule refused, over both splits: a rule code +/// (`V5`, `UnknownEntity`) and a count, never an identifier or text. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct RuleCount { + pub rule: String, + pub count: u32, +} + /// A Train item's row. Validation has no rows. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct TrainRow { @@ -212,6 +223,7 @@ pub struct RefusalCounts { /// validation_refused: RefusalCounts::default(), /// inventory_digest: String::new(), /// cycle: None, +/// admit_rules: vec![], /// } /// } /// ``` @@ -262,6 +274,10 @@ pub struct RoundRecord { /// `Some(k)` for the candidate round of cycle `k`. #[cultcache(key = 17)] cycle: Option, + /// The rules admission refused items under, in rule order; every Train + /// and Validation item refused at `Stage::Admit` is counted once per rule. + #[cultcache(key = 18)] + admit_rules: Vec, } impl RoundRecord { @@ -316,6 +332,9 @@ impl RoundRecord { pub fn cycle(&self) -> Option { self.cycle } + pub fn admit_rules(&self) -> &[RuleCount] { + &self.admit_rules + } /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -333,7 +352,7 @@ impl RoundRecord { &self.validation_mean, self.validation_refused, ); - format!( + let line = format!( "round {} rev {} train n={} scored={} chrf={} meaning={} refused={refused} validation n={} scored={} chrf={} meaning={} refused parse={} admit={} lower={}", self.n, self.inventory_rev, @@ -348,7 +367,17 @@ impl RoundRecord { x.parse, x.admit, x.lower, - ) + ); + if self.admit_rules.is_empty() { + line + } else { + let rules: Vec = self + .admit_rules + .iter() + .map(|r| format!("{}x{}", r.rule, r.count)) + .collect(); + format!("{line} admit-rules={}", rules.join(",")) + } } } @@ -772,6 +801,25 @@ pub enum RoundOutcome { Recorded(RoundRecord), } +/// The rule codes an admission refusal broke, each once: validator rules by +/// code, a store refusal by its variant name. Nothing from the packet or the +/// stores is kept; an I/O failure is `io`. +fn admit_rule_codes(e: &StoreError) -> Vec { + let mut codes: Vec = match e { + StoreError::Invalid(rs) => rs.iter().map(|r| r.rule().to_string()).collect(), + StoreError::Refused(r) => vec![r + .to_string() + .split([' ', ':']) + .next() + .unwrap_or_default() + .to_string()], + StoreError::Io(_) => vec!["io".to_string()], + }; + codes.sort(); + codes.dedup(); + codes +} + /// How one item fared: where it was refused, or the text it lowered to, with /// the key of the packet that was admitted for it. enum Fate { @@ -876,6 +924,7 @@ fn score_round( let encoder = encoder_id(model.model(), &run.record.prompt_digest); let mut items: Vec = Vec::new(); let mut fates: Vec = Vec::new(); + let mut admit_rules: BTreeMap = BTreeMap::new(); // Phase 1 ends at admission: each admitted view waits for its lowering. let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new(); for (key, answer) in &answers { @@ -887,7 +936,12 @@ fn score_round( let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); match packets.admit(packet, inventory_store, &run.corpus, &run.entities) { - Err(_) => Fate::Refused(Stage::Admit, None), + Err(e) => { + for rule in admit_rule_codes(&e) { + *admit_rules.entry(rule).or_default() += 1; + } + Fate::Refused(Stage::Admit, None) + } Ok(view) => { admitted.push((fates.len(), id, packet_key, view)); // Replaced below by what the lowering answers. @@ -1038,6 +1092,10 @@ fn score_round( validation_scored, inventory_digest: inventory_digest.to_string(), cycle, + admit_rules: admit_rules + .into_iter() + .map(|(rule, count)| RuleCount { rule, count }) + .collect(), }; Ok(RoundOutcome::Recorded(record)) } @@ -1470,6 +1528,7 @@ mod tests { validation_refused: RefusalCounts::default(), inventory_digest: String::new(), cycle: None, + admit_rules: vec![], }; assert_eq!(r.n(), 1); } @@ -1685,6 +1744,7 @@ mod tests { validation_refused: RefusalCounts::default(), inventory_digest: digest.to_string(), cycle: None, + admit_rules: vec![], } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index a4bba52..4b18427 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2898,3 +2898,41 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { unique.dedup(); assert_eq!(unique.len(), 2); } + +#[test] +fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { + use weksa::packet::EntityId; + let mut dao = dao(); + let empty = |d: &mut PacketDraft| { + d.referents.clear(); + d.predications.clear(); + d.relations.clear(); + }; + let nobody = |d: &mut PacketDraft| { + d.referents[0].entity = Some("entity:nobody".parse::().unwrap()); + }; + let mut model = fake(&[ + (1, edited(1, nobody)), + (2, edited(2, unknown_concept)), + (3, edited(3, empty)), + ]); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rules: Vec<(&str, u32)> = r + .admit_rules() + .iter() + .map(|c| (c.rule.as_str(), c.count)) + .collect(); + assert_eq!(rules, [("UnknownEntity", 1), ("V2", 1), ("V5", 1)]); + let line = dao.run.round_line(&r); + assert!( + line.ends_with("admit-rules=UnknownEntityx1,V2x1,V5x1 lowering=tool-less-relay"), + "{line}" + ); + + // A round nothing refused names no rule and prints none. + let mut dao = self::dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert!(r.admit_rules().is_empty()); + assert!(!dao.run.round_line(&r).contains("admit-rules")); +} diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index 27205d1..dd940ad 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,9 +1,16 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```", "type": "object", "properties": { + "admit_rules": { + "description": "The rules admission refused items under, in rule order; every Train\nand Validation item refused at `Stage::Admit` is counted once per rule.", + "type": "array", + "items": { + "$ref": "#/$defs/RuleCount" + } + }, "calls": { "description": "The encode calls, then the lowering calls, each with its prompt digest\nand one item key.", "type": "array", @@ -96,7 +103,8 @@ "validation_refused", "train_scored", "validation_scored", - "inventory_digest" + "inventory_digest", + "admit_rules" ], "$defs": { "Call": { @@ -187,6 +195,24 @@ "lower" ] }, + "RuleCount": { + "description": "How many items one admission rule refused, over both splits: a rule code\n(`V5`, `UnknownEntity`) and a count, never an identifier or text.", + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "rule": { + "type": "string" + } + }, + "required": [ + "rule", + "count" + ] + }, "SplitMean": { "description": "Mean scores of one derived split.", "type": "object", From d502cb396a657f62ec9231f1799836e187aee1ba Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:08:46 +0000 Subject: [PATCH 04/93] Admit rules are a set: one count per item per rule, pinned by a packet that breaks V5 many times Soul s1 low: the per-item dedup was unpinned. A BTreeSet makes it structural. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 20 +++++++++----------- crates/weksa/tests/golf_loop.rs | 12 +++++++++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 93ae120..299ea72 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -23,7 +23,7 @@ //! uses the template table. The prompt source pin covers `context.rs`, //! `split.rs` and `lower.rs`. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::io::ErrorKind; use std::path::{Path, PathBuf}; @@ -801,23 +801,21 @@ pub enum RoundOutcome { Recorded(RoundRecord), } -/// The rule codes an admission refusal broke, each once: validator rules by +/// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. -fn admit_rule_codes(e: &StoreError) -> Vec { - let mut codes: Vec = match e { +fn admit_rule_codes(e: &StoreError) -> BTreeSet { + match e { StoreError::Invalid(rs) => rs.iter().map(|r| r.rule().to_string()).collect(), - StoreError::Refused(r) => vec![r + StoreError::Refused(r) => [r .to_string() .split([' ', ':']) .next() .unwrap_or_default() - .to_string()], - StoreError::Io(_) => vec!["io".to_string()], - }; - codes.sort(); - codes.dedup(); - codes + .to_string()] + .into(), + StoreError::Io(_) => ["io".to_string()].into(), + } } /// How one item fared: where it was refused, or the text it lowered to, with diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 4b18427..6269f99 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2899,6 +2899,16 @@ fn views_with_one_meaning_share_one_unanswered_lowering_prompt() { assert_eq!(unique.len(), 2); } +/// Several refusals under one rule in one packet: the item still counts once. +fn every_concept_unknown(d: &mut PacketDraft) { + for r in &mut d.referents { + r.concept = "concept:999".parse().unwrap(); + } + for p in &mut d.predications { + p.concept = "concept:999".parse().unwrap(); + } +} + #[test] fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { use weksa::packet::EntityId; @@ -2913,7 +2923,7 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { }; let mut model = fake(&[ (1, edited(1, nobody)), - (2, edited(2, unknown_concept)), + (2, edited(2, every_concept_unknown)), (3, edited(3, empty)), ]); let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); From d954cfb9e961f1948db5f94d0053c5512a5994e3 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:12:30 +0000 Subject: [PATCH 05/93] weksa loop recall: a recitation probe for public-domain items; admit_rules not required in the round schema Each live public-domain Train and Validation item is offered to a model by its opening alone and the completion is scored by chrF against the item's second half (lowering-recall-diagnostic). Answers live in /recall; the reading is printed, recorded nowhere, never a gate. The round schema stops requiring admit_rules, which older records lack. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 20 ++++ crates/weksa/src/main.rs | 21 +++- crates/weksa/src/run.rs | 104 ++++++++++++++++++- crates/weksa/tests/golf_loop.rs | 93 +++++++++++++++++ schemas/cultnet/weksa.loop_round.schema.json | 4 +- 5 files changed, 237 insertions(+), 5 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index dc1e19f..e45e941 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -70,6 +70,13 @@ Every id is the inventory's listed below and every feature value is as listed. T A referent with an entity is named by one of that entity's forms on the entities line. Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; +/// The instructions of a recall probe (`lowering-recall-diagnostic`): a +/// completion of a public-domain passage from its opening, asked of a model +/// that has no packet and no inventory. It measures recitation, not lowering. +pub const RECALL_INSTRUCTIONS: &str = "\ +Below is the opening of a passage from a published public-domain translation. Continue it exactly as that translation words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. +Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuation only, not the opening. No code fence, no commentary."; + /// The live Train items of a corpus, in key order. The only input minting /// reads; it cannot be built from a slice, and no accessor returns text or an /// item. @@ -540,6 +547,19 @@ impl Prompt { }) } + /// A recall probe: the work, author and translator of a public-domain item + /// and the `opening` of its text, nothing else. Only `RunStores::recall` + /// builds one. + pub(crate) fn recall(key: &str, work: &str, author: &str, translator: &str, opening: &str) -> Self { + Prompt { + text: format!( + "{RECALL_INSTRUCTIONS}\n\nwork: {work}\nauthor: {author}\ntranslator: {translator}\n\nopening: {}", + indented(opening) + ), + items: vec![key.to_string()], + } + } + /// 16 lowercase hex digits of FNV-1a 64 of the text: the name of the prompt /// in files and records. pub fn digest(&self) -> String { diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 06b3cce..4b65ce6 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -11,7 +11,7 @@ use weksa::corpus::{CorpusItem, ModelFamily, NameSpan, Source, Turn}; use weksa::entity::{Entity, EntityOrigin, LanguageForms, Pronunciation}; use weksa::inventory::{proof_revision, seed}; use weksa::packet::proof_packet; -use weksa::run::{CycleOutcome, LowererReach, RoundOutcome, RunPaths, RunStores, cycle, round}; +use weksa::run::{CycleOutcome, LowererReach, RecallOutcome, RoundOutcome, RunPaths, RunStores, cycle, round}; use weksa::score::ScoreInput; use weksa::{ AnswerFiles, CorpusStore, EntityId, EntityStore, InventoryStore, OllamaEmbedder, PacketStore, @@ -103,6 +103,11 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, + /// Probe how closely a fresh model recites the run's public-domain items + /// from their openings (chrF of its completion against the item's second + /// half). Prompts and answers live in `/recall`; the command exits 3 + /// until every prompt has an answer. A diagnostic, never a gate. + Recall { root: PathBuf }, /// Run one edit-accept cycle of the run at ROOT. The stores are read only /// from the run's record. The editor's prompt, then the candidate round's /// encode prompts, then its lowering prompts are written in turn; the @@ -774,6 +779,20 @@ fn main() -> Result<()> { RoundOutcome::Recorded(r) => println!("{}", run.round_line(&r)), } } + Command::Loop(LoopCommand::Recall { root }) => { + let run = RunStores::open(&root)?; + let mut files = AnswerFiles::new(run.recall_dir(), run.record().model.clone()); + match run.recall(&mut files)? { + RecallOutcome::Unanswered { prompts } => { + for digest in &prompts { + println!("unanswered {digest}"); + } + println!("unanswered {} prompts", prompts.len()); + std::process::exit(3); + } + RecallOutcome::Read(reading) => println!("{}", reading.line()), + } + } Command::Loop(LoopCommand::Cycle { root, ollama_url }) => { let unreadable = |_| anyhow::anyhow!("a store would not read"); let mut run = RunStores::open(&root)?; diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 299ea72..3ae47dd 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -35,11 +35,11 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; -use crate::corpus::{CorpusItem, CorpusItemId}; +use crate::corpus::{CorpusItem, CorpusItemId, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; -use crate::score::{CHRF_VERSION, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; +use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate}; use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16}; use crate::store::{ CorpusStore, EntityStore, InventoryStore, PacketStore, StoreError, open_cache_over, @@ -277,6 +277,7 @@ pub struct RoundRecord { /// The rules admission refused items under, in rule order; every Train /// and Validation item refused at `Stage::Admit` is counted once per rule. #[cultcache(key = 18)] + #[schemars(default)] admit_rules: Vec, } @@ -801,6 +802,105 @@ pub enum RoundOutcome { Recorded(RoundRecord), } +/// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model +/// completes each public-domain item from its opening alone, chrF against the +/// item's own second half. Recorded nowhere: the answers stay in `/recall`. +/// It is never a gate; a high reading says a round's scores on these items are +/// recitation. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallReading { + /// Public-domain items probed, Train and Validation. + pub n: u32, + /// Items whose completion was a usable answer; a refused one counts 0. + pub answered: u32, + pub chrf: f64, +} + +impl RecallReading { + pub fn line(&self) -> String { + format!( + "recall n={} answered={} chrf={:.2} (public-domain items, completion from the opening; a high chrf means round scores on them are recitation)", + self.n, self.answered, self.chrf + ) + } +} + +#[derive(Debug)] +pub enum RecallOutcome { + Unanswered { prompts: Vec }, + Read(RecallReading), +} + +/// The opening and the rest of `text`, split between words at about the +/// middle; `None` when it has fewer than four words. +fn recall_split(text: &str) -> Option<(String, String)> { + let words: Vec<&str> = text.split_whitespace().collect(); + if words.len() < 4 { + return None; + } + let (head, tail) = words.split_at(words.len() / 2); + Some((head.join(" "), tail.join(" "))) +} + +impl RunStores { + /// Probes every live public-domain Train and Validation item of the pinned + /// corpus once, through `model`. + pub fn recall(&self, model: &mut dyn ModelPort) -> Result { + let store = |_| RunError::Store; + let mut probes = Vec::new(); + for item in self.corpus.all().map_err(store)? { + let Source::PublicDomain { + work, + author, + translator, + .. + } = &item.source + else { + continue; + }; + if EncoderOffer::for_round(&item).is_err() { + continue; + } + if let Some((opening, rest)) = recall_split(&item.text) { + let prompt = Prompt::recall(&item.id, work, author, translator, &opening); + probes.push((prompt, rest)); + } + } + let mut unanswered = Vec::new(); + let mut scores = Vec::new(); + for (prompt, rest) in &probes { + match model.answer(prompt) { + Ok(answer) => { + let text = LoweringDraft::parse(&answer) + .ok() + .filter(LoweringDraft::usable) + .map(|d| d.text); + scores.push(text.map(|t| chrf(&t, rest))); + } + Err(PortError::Unanswered) => unanswered.push(prompt.digest()), + Err(PortError::Io(ErrorKind::InvalidData)) => scores.push(None), + Err(PortError::Io(kind)) => return Err(RunError::Io(kind)), + } + } + if !unanswered.is_empty() { + return Ok(RecallOutcome::Unanswered { prompts: unanswered }); + } + let n = scores.len() as u32; + let answered = scores.iter().flatten().count() as u32; + let chrf = if n == 0 { + 0.0 + } else { + scores.iter().map(|s| s.unwrap_or(0.0)).sum::() / f64::from(n) + }; + Ok(RecallOutcome::Read(RecallReading { n, answered, chrf })) + } + + /// `/recall`, where a recall's prompts and answers live. + pub fn recall_dir(&self) -> PathBuf { + self.root.join("recall") + } +} + /// The distinct rule codes an admission refusal broke: validator rules by /// code, a store refusal by its variant name. Nothing from the packet or the /// stores is kept; an I/O failure is `io`. diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 6269f99..e52da20 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2946,3 +2946,96 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { assert!(r.admit_rules().is_empty()); assert!(!dao.run.round_line(&r).contains("admit-rules")); } + +/// Answers a recall probe from the corpus: the item's own second half, a +/// fixed wrong text, or junk. +struct Reciter { + corpus: weksa::store::CorpusStore, + answer: fn(&str) -> String, + seen: Vec, +} + +impl ModelPort for Reciter { + fn model(&self) -> &str { + "reciter" + } + fn answer(&mut self, prompt: &Prompt) -> Result { + self.seen.push(prompt.text().to_string()); + let key = &prompt.item_keys()[0]; + let item = self.corpus.get(key).unwrap().unwrap(); + Ok((self.answer)(&item.text)) + } +} + +fn second_half(text: &str) -> String { + let words: Vec<&str> = text.split_whitespace().collect(); + let draft = serde_json::json!({ "text": words[words.len() / 2..].join(" ") }); + draft.to_string() +} + +fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { + Reciter { + corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(), + answer, + seen: vec![], + } +} + +#[test] +fn a_recall_probe_asks_for_the_second_half_of_each_live_public_domain_item_and_never_shows_it() { + use weksa::run::RecallOutcome; + let dao = dao(); + let mut perfect = reciter(&dao, second_half); + let RecallOutcome::Read(r) = dao.run.recall(&mut perfect).unwrap() else { + panic!("every prompt was answered"); + }; + // Items 1 (Train) and 2, 3 (Validation) are live; 4 and 5 are not offered. + assert_eq!((r.n, r.answered), (3, 3)); + assert!((r.chrf - 100.0).abs() < 1e-9, "{}", r.chrf); + assert_eq!(perfect.seen.len(), 3); + for (text, key) in perfect.seen.iter().zip(["item:daodejing:1", "item:daodejing:2", "item:daodejing:3"]) { + let item = perfect.corpus.get(key).unwrap().unwrap(); + let rest = item.text.split_whitespace().collect::>(); + let tail = rest[rest.len() / 2..].join(" "); + let head = rest[..rest.len() / 2].join(" "); + assert!(text.contains(&head), "{key}: the opening is shown"); + assert!(!text.contains(&tail), "{key}: the held half is not shown"); + } +} + +#[test] +fn a_recall_of_the_wrong_words_or_no_answer_reads_low_and_a_missing_answer_waits() { + use weksa::run::RecallOutcome; + let dao = dao(); + let mut wrong = reciter(&dao, |_| r#"{"text":"zzz qqq xxx"}"#.to_string()); + let RecallOutcome::Read(r) = dao.run.recall(&mut wrong).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered), (3, 3)); + assert!(r.chrf < 10.0, "{}", r.chrf); + let mut junk = reciter(&dao, |_| "not json".to_string()); + let RecallOutcome::Read(r) = dao.run.recall(&mut junk).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered, r.chrf), (3, 0, 0.0)); + // With no answers on disk the files port waits on all three prompts. + let mut files = AnswerFiles::new(dao.run.recall_dir(), "m".into()); + let RecallOutcome::Unanswered { prompts } = dao.run.recall(&mut files).unwrap() else { + panic!("nobody answered"); + }; + assert_eq!(prompts.len(), 3); + assert!(dao.run.recall_dir().join(format!("{}.prompt", prompts[0])).exists()); +} + +#[test] +fn the_published_round_schema_does_not_require_a_field_older_records_lack() { + let (_, json) = weksa::schema::published() + .into_iter() + .find(|(name, _)| name.contains("loop_round")) + .expect("the round schema is published"); + let schema: serde_json::Value = serde_json::from_str(&json).unwrap(); + let required = schema["required"].as_array().unwrap(); + assert!(required.iter().any(|r| r == "inventory_digest")); + assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}"); + assert!(schema["properties"]["admit_rules"].is_object()); +} diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index dd940ad..f218bdb 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -7,6 +7,7 @@ "admit_rules": { "description": "The rules admission refused items under, in rule order; every Train\nand Validation item refused at `Stage::Admit` is counted once per rule.", "type": "array", + "default": [], "items": { "$ref": "#/$defs/RuleCount" } @@ -103,8 +104,7 @@ "validation_refused", "train_scored", "validation_scored", - "inventory_digest", - "admit_rules" + "inventory_digest" ], "$defs": { "Call": { From 0ead162d8e24fd8d9640def4252a1f8a9b6cb4e0 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:15:27 +0000 Subject: [PATCH 06/93] Recall tests: mean over every probe, unreadable answer scores 0, written and withdrawn items are not probed Soul lows: the mean-to-max, skipped-for-0 and withdrawn-gate mutants survived. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/tests/golf_loop.rs | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index e52da20..ae0b7fe 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2952,6 +2952,8 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { struct Reciter { corpus: weksa::store::CorpusStore, answer: fn(&str) -> String, + /// An item whose text holds this gets an answer file that is not UTF-8. + unreadable: Option<&'static str>, seen: Vec, } @@ -2963,6 +2965,9 @@ impl ModelPort for Reciter { self.seen.push(prompt.text().to_string()); let key = &prompt.item_keys()[0]; let item = self.corpus.get(key).unwrap().unwrap(); + if self.unreadable.is_some_and(|t| item.text.contains(t)) { + return Err(PortError::Io(std::io::ErrorKind::InvalidData)); + } Ok((self.answer)(&item.text)) } } @@ -2977,6 +2982,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { Reciter { corpus: weksa::store::CorpusStore::open(&dao.corpus).unwrap(), answer, + unreadable: None, seen: vec![], } } @@ -3039,3 +3045,40 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() { assert!(!required.iter().any(|r| r == "admit_rules"), "{required:?}"); assert!(schema["properties"]["admit_rules"].is_object()); } + +#[test] +fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() { + use weksa::corpus::Source; + use weksa::run::RecallOutcome; + let dao = dao_over(|corpus, entities| { + let entities = weksa::EntityStore::open(entities).unwrap(); + let mut store = weksa::CorpusStore::open(corpus).unwrap(); + // The same words under other sources: not public domain, not probed. + let mut written = store.get("item:daodejing:2").unwrap().unwrap(); + written.id = "item:written:1".into(); + written.text = "A line of some other source with words enough to split.".into(); + written.names = vec![]; + written.prior_turns = vec![]; + written.source = Source::OperatorWritten; + store.add(written, &entities).unwrap(); + store.withdraw("item:daodejing:3").unwrap(); + }); + let both = |dao: &Dao, unreadable: Option<&'static str>| { + let mut m = reciter(dao, second_half); + m.unreadable = unreadable; + let RecallOutcome::Read(r) = dao.run.recall(&mut m).unwrap() else { + panic!("answered"); + }; + r + }; + let full = both(&dao, None); + assert_eq!( + (full.n, full.answered), + (2, 2), + "neither the written item nor the withdrawn one is probed" + ); + // One of two unreadable: the mean falls to a half. + let part = both(&dao, Some("Heaven is long-enduring")); + assert_eq!((part.n, part.answered), (2, 1)); + assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf); +} From 1c08b2abaccde7fcc01b75ae9df715e13ea3b06b Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:18:28 +0000 Subject: [PATCH 07/93] Original corpus (data/original): six passages written for the non-memorised target; recall probes every live item Six short original sentences, model-written (Anthropic, claude-sonnet-5-5) and so labelled, three Train and three Validation, over the Dao revision-3 inventory and one live entity. The recall probe now covers every offered item, naming the work only for a public-domain one, so it can show the original set reads near the guessing floor. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 16 +++--- crates/weksa/src/main.rs | 4 +- crates/weksa/src/run.rs | 29 +++++------ crates/weksa/tests/golf_loop.rs | 22 ++++++--- crates/weksa/tests/original_corpus.rs | 67 ++++++++++++++++++++++++++ data/original/corpus.cc | 1 + data/original/entity.cc | 1 + data/original/inventory.cc | Bin 0 -> 10559 bytes 8 files changed, 108 insertions(+), 32 deletions(-) create mode 100644 crates/weksa/tests/original_corpus.rs create mode 100644 data/original/corpus.cc create mode 100644 data/original/entity.cc create mode 100644 data/original/inventory.cc diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index e45e941..da6262d 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -71,10 +71,10 @@ A referent with an entity is named by one of that entity's forms on the entities Answer with one JSON object and nothing else: {\"text\": \"...\"}. No code fence, no commentary."; /// The instructions of a recall probe (`lowering-recall-diagnostic`): a -/// completion of a public-domain passage from its opening, asked of a model -/// that has no packet and no inventory. It measures recitation, not lowering. +/// completion of a passage from its opening, asked of a model that has no +/// packet and no inventory. It measures recitation, not lowering. pub const RECALL_INSTRUCTIONS: &str = "\ -Below is the opening of a passage from a published public-domain translation. Continue it exactly as that translation words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. +Below is the opening of a passage, with what is known of where it comes from. Continue it exactly as its source words it, from memory, until the passage ends. If you do not remember it, give your best guess in the same voice. Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuation only, not the opening. No code fence, no commentary."; /// The live Train items of a corpus, in key order. The only input minting @@ -547,13 +547,13 @@ impl Prompt { }) } - /// A recall probe: the work, author and translator of a public-domain item - /// and the `opening` of its text, nothing else. Only `RunStores::recall` - /// builds one. - pub(crate) fn recall(key: &str, work: &str, author: &str, translator: &str, opening: &str) -> Self { + /// A recall probe: `provenance` (header lines saying where the item is + /// published, or that it is not) and the `opening` of its text, nothing + /// else. Only `RunStores::recall` builds one. + pub(crate) fn recall(key: &str, provenance: &str, opening: &str) -> Self { Prompt { text: format!( - "{RECALL_INSTRUCTIONS}\n\nwork: {work}\nauthor: {author}\ntranslator: {translator}\n\nopening: {}", + "{RECALL_INSTRUCTIONS}\n\n{provenance}\n\nopening: {}", indented(opening) ), items: vec![key.to_string()], diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 4b65ce6..3808672 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -103,8 +103,8 @@ enum LoopCommand { #[arg(long, default_value = "http://127.0.0.1:11434")] ollama_url: String, }, - /// Probe how closely a fresh model recites the run's public-domain items - /// from their openings (chrF of its completion against the item's second + /// Probe how closely a fresh model recites the run's items from their + /// openings (chrF of its completion against the item's second /// half). Prompts and answers live in `/recall`; the command exits 3 /// until every prompt has an answer. A diagnostic, never a gate. Recall { root: PathBuf }, diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 3ae47dd..0d8f1d7 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -803,13 +803,13 @@ pub enum RoundOutcome { } /// A recall reading (`lowering-recall-diagnostic`): how closely a fresh model -/// completes each public-domain item from its opening alone, chrF against the +/// completes each live Train and Validation item from its opening alone, chrF against the /// item's own second half. Recorded nowhere: the answers stay in `/recall`. /// It is never a gate; a high reading says a round's scores on these items are /// recitation. #[derive(Clone, Debug, PartialEq)] pub struct RecallReading { - /// Public-domain items probed, Train and Validation. + /// Items probed, Train and Validation. pub n: u32, /// Items whose completion was a usable answer; a refused one counts 0. pub answered: u32, @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (public-domain items, completion from the opening; a high chrf means round scores on them are recitation)", + "recall n={} answered={} chrf={:.2} (completion from the opening; a high chrf means round scores on these items are recitation; unrelated words score near 0 and an in-voice guess about 20)", self.n, self.answered, self.chrf ) } @@ -843,26 +843,27 @@ fn recall_split(text: &str) -> Option<(String, String)> { } impl RunStores { - /// Probes every live public-domain Train and Validation item of the pinned - /// corpus once, through `model`. + /// Probes every live Train and Validation item of the pinned corpus once, + /// through `model`. A public-domain item names its work, author and + /// translator; any other says `work: not published`. pub fn recall(&self, model: &mut dyn ModelPort) -> Result { let store = |_| RunError::Store; let mut probes = Vec::new(); for item in self.corpus.all().map_err(store)? { - let Source::PublicDomain { - work, - author, - translator, - .. - } = &item.source - else { - continue; + let provenance = match &item.source { + Source::PublicDomain { + work, + author, + translator, + .. + } => format!("work: {work}\nauthor: {author}\ntranslator: {translator}"), + _ => "work: not published".to_string(), }; if EncoderOffer::for_round(&item).is_err() { continue; } if let Some((opening, rest)) = recall_split(&item.text) { - let prompt = Prompt::recall(&item.id, work, author, translator, &opening); + let prompt = Prompt::recall(&item.id, &provenance, &opening); probes.push((prompt, rest)); } } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index ae0b7fe..4d9dc1c 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -2988,7 +2988,7 @@ fn reciter(dao: &Dao, answer: fn(&str) -> String) -> Reciter { } #[test] -fn a_recall_probe_asks_for_the_second_half_of_each_live_public_domain_item_and_never_shows_it() { +fn a_recall_probe_asks_for_the_second_half_of_each_live_item_and_never_shows_it() { use weksa::run::RecallOutcome; let dao = dao(); let mut perfect = reciter(&dao, second_half); @@ -3047,13 +3047,13 @@ fn the_published_round_schema_does_not_require_a_field_older_records_lack() { } #[test] -fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live_public_domain_items_are_probed() { +fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_a_withdrawn_item_is_not_probed() { use weksa::corpus::Source; use weksa::run::RecallOutcome; let dao = dao_over(|corpus, entities| { let entities = weksa::EntityStore::open(entities).unwrap(); let mut store = weksa::CorpusStore::open(corpus).unwrap(); - // The same words under other sources: not public domain, not probed. + // Another source: probed too, but it names no work. let mut written = store.get("item:daodejing:2").unwrap().unwrap(); written.id = "item:written:1".into(); written.text = "A line of some other source with words enough to split.".into(); @@ -3074,11 +3074,17 @@ fn a_recall_mean_is_over_every_probe_an_unreadable_answer_counts_0_and_only_live let full = both(&dao, None); assert_eq!( (full.n, full.answered), - (2, 2), - "neither the written item nor the withdrawn one is probed" + (3, 3), + "the written item is probed, the withdrawn one is not" ); - // One of two unreadable: the mean falls to a half. + // One of three unreadable: the mean falls to two thirds. let part = both(&dao, Some("Heaven is long-enduring")); - assert_eq!((part.n, part.answered), (2, 1)); - assert!((part.chrf - full.chrf / 2.0).abs() < 1e-9, "{}", part.chrf); + assert_eq!((part.n, part.answered), (3, 2)); + assert!((part.chrf - full.chrf * 2.0 / 3.0).abs() < 1e-9, "{}", part.chrf); + // A public-domain prompt names its work; any other says it is not published. + let mut m = reciter(&dao, second_half); + dao.run.recall(&mut m).unwrap(); + let named = m.seen.iter().filter(|t| t.contains("translator: James Legge")).count(); + let unnamed = m.seen.iter().filter(|t| t.contains("work: not published")).count(); + assert_eq!((named, unnamed), (2, 1)); } diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs new file mode 100644 index 0000000..4416a36 --- /dev/null +++ b/crates/weksa/tests/original_corpus.rs @@ -0,0 +1,67 @@ +//! The original corpus (`data/original`): passages written for the +//! non-memorised golf-loop target. It holds no public-domain text, so a model +//! cannot have it in pretraining, and every item says who wrote it. + +use std::path::Path; + +use weksa::corpus::{ModelFamily, Source}; +use weksa::split::{Split, SplitScheme}; +use weksa::{CorpusStore, EntityStore, InventoryStore}; + +fn stores() -> (CorpusStore, EntityStore) { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(dir.join(f), tmp.path().join(f)).unwrap(); + } + let keep = tmp.keep(); + ( + CorpusStore::open(&keep.join("corpus.cc")).unwrap(), + EntityStore::open(&keep.join("entity.cc")).unwrap(), + ) +} + +#[test] +fn every_original_item_records_its_author_and_none_is_public_domain() { + let (corpus, entities) = stores(); + let items = corpus.all().unwrap(); + assert_eq!(items.len(), 6); + for item in &items { + match &item.source { + Source::ModelWritten { + family, generator, .. + } => { + assert_eq!(*family, ModelFamily::Anthropic, "{}", item.id); + let generator = generator.as_deref().unwrap_or_default(); + assert!(generator.contains("claude-sonnet-5-5"), "{}", item.id); + } + other => panic!("{}: not model-written: {other:?}", item.id), + } + assert!(entities.get(&item.speaker).unwrap().is_some(), "{}", item.id); + } +} + +#[test] +fn the_original_items_spread_over_train_and_validation_and_none_is_in_test() { + let (corpus, _) = stores(); + let splits: Vec> = corpus + .all() + .unwrap() + .iter() + .map(|i| SplitScheme::V1.live(i)) + .collect(); + let count = |s| splits.iter().filter(|x| **x == Some(s)).count(); + assert_eq!((count(Split::Train), count(Split::Validation)), (3, 3)); + assert_eq!(count(Split::Test), 0); +} + +#[test] +fn the_original_inventory_is_the_dao_revision_three() { + let dao = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/daodejing/inventory.cc"); + let own = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original/inventory.cc"); + assert_eq!(std::fs::read(dao).unwrap(), std::fs::read(&own).unwrap()); + let tmp = tempfile::tempdir().unwrap(); + let copy = tmp.path().join("inventory.cc"); + std::fs::copy(&own, ©).unwrap(); + assert!(InventoryStore::open(©).is_ok()); +} diff --git a/data/original/corpus.cc b/data/original/corpus.cc new file mode 100644 index 0000000..c11443f --- /dev/null +++ b/data/original/corpus.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item�–”¯item:original:1±weksa.corpus_item´2026-11-10T14:17:08ZÄ훯item:original:1Ù0A quiet mind is like still water in a deep bowl.¥en-US­entity:keeper­original-bowl��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:2±weksa.corpus_item´2026-11-10T14:17:08ZÅ›¯item:original:2ÙIThe river weakens the bank in spring and strengthens the roots in autumn.¥en-US­entity:keeper®original-river��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:3±weksa.corpus_item´2026-11-10T14:17:08ZÄô›¯item:original:3Ù7The sage waits at the door until the water has settled.¥en-US­entity:keeper­original-door��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:4±weksa.corpus_item´2026-11-10T14:17:08ZÄÿ›¯item:original:4ÙCThe old gardener empties the jar and fills it again with rainwater.¥en-US­entity:keeper¬original-jar��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:5±weksa.corpus_item´2026-11-10T14:17:08ZÅ›¯item:original:5ÙJUnder the same sky the earth keeps its patience, and the baker keeps hers.¥en-US­entity:keeper¬original-sky��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ�”¯item:original:6±weksa.corpus_item´2026-11-10T14:17:08ZÄ雯item:original:6Ù+A hungry belly argues louder than any will.¥en-US­entity:keeper®original-belly��¬ModelWritten“©AnthropicÙ]claude-sonnet-5-5 (Weksa Self, original prose written for the non-memorised golf-loop target)Àª2026-11-10ÀÂ� \ No newline at end of file diff --git a/data/original/entity.cc b/data/original/entity.cc new file mode 100644 index 0000000..c92e33d --- /dev/null +++ b/data/original/entity.cc @@ -0,0 +1 @@ +“²cultcache.store.v1‘—¬weksa.entity¬weksa.entity¯weksa.entity.v1¬weksa.entityÙL{"schemaName":"weksa.entity","schemaVersion":"weksa.entity.v1","members":[]}‘¬weksa.entity�‘”­entity:keeper¬weksa.entity´2026-11-10T14:17:08ZÄ:“­entity:keeper‘“¥en-US‘ªthe keeperÀ�¤Live‘®weksa-original \ No newline at end of file diff --git a/data/original/inventory.cc b/data/original/inventory.cc new file mode 100644 index 0000000000000000000000000000000000000000..373eb3cf259663321fdf94128b4f11a984998f01 GIT binary patch literal 10559 zcmb7KTaP1072eA)2xMn=ClfY1T%O2g+`i682q7Wz21r3XfS@UNm+fZS-KMHNUW0hZ z3(vdD#1YTy_GM;CmMBO_NN5!$KO)lp7R2eU>Z~b4JfNc^E|JGy><`g@0^b@BLTpvj9{7_}9Pv(-DW>#^k4D zOpi{FYMUQ@Uq19p%6Je*yPkt?;L$N1_u;vt(_jDQ&rj;Ref)8SC!^;e?uQLXdIt5 zI`H?aFYhjNyA6$e5)CPP`S{k$i%W%fTtB__fOi%~!{sap!V8yk63lc zL5cG{V`5wX;S=Ifw@;H9rI9OS;F8F9YckkDiT1A1PkY^mu*EZbN>GY50Xu6i9>av# zX@HXKJ%eF)!zhP!I24FV1E`?wX$>@2laS2i`lHZLfie`;g5kmzCWgSna;D%=@wlhS zwYPi}Ox$r0`DH*;0X)G4n?SPVG$|!D+^23F0x0%EtVL>wgaQ(AM0y{bW6kI$=f^)iw=#XR|o?QK$ zQ6Ds)J#Z5?AF5kaHHC7b8!q7^OW%iTLZ7TYNC;HZ5;Cfme#by=tU2)`%K0_msLJ|| z0q#8eeLTMg7^PH`b#v_pd^qLFHQ*@We%D~#+VClxuz=HR&`~-z(c7D>E@ZpEYtT`m z{+_|Tvn7K8^|H*n1|B8yBLlvh@)4Y&u7O5r-elg}LN!kq38L!6C;_4}@qH~^CrxO? zsZ#C5azc2L?Rhxtc~*mIRu(o%Fs30$!4|OU9+j`Z)7>{$gijzUs{fBJCzR!JI;YQr zu{N$IEFJ z%5Ed-0W-qpYoGD}YTP#NhXE8l>hGk6(#Z#!TKn&CR`vx0li$V4hn~G;6GQ{bfCFp- zCNTpFYr^?-d<_+psE#$2PCBCgP$vD;?T-WQ#Z$(qRpwQepw!jU{dBh;Wl1IgSuia-zcV~n} z`N;}4Sn@kmfjB4-o#!y=lx621D4hBJO5&jue^lT#KbLzecNZ>7v2PT(t!E(4{D>my zq12^@+s=l|NF??xCLT&6dj+k|CJf-Z0BVDYg_6Pj0;~Iv?BK*JCnq^zRKCAmfc8Ee zlR2E2TzLY4Ei~*cKV=k32*lhygIzT?8SU09DAGJj9@v+?u#i#vzQaYOc`A?QHPEPy z-~9@7V?+Acs+*-1pj(4Nsp+kS-8vnzcsi*AM#<`1HgIzjvmuEBKCZ(?NvlG-mBJw~ zyQ-+p21;xd((Mi0F9m}gGmR471A9U`S3$%R*ei(gRi%#7;@dWScTK%GjE8WFszRd_ zS()x$3MrD*PB&`JIkY*Stl)|s-Xr9cFBuK<(->CdM#C#Fma3N7aiu{~`aW2gEjO?S z&DI?#oiMk?Hz;B6nV5}bJc}xLC@q)p9KfR$Jd}j@O@7VvEXwbd$^%B}wglKpBiKz; z7^3ugXfkX=+~wJzf{Ie=J0@x;zqw^_@Zjz&$nKh90(6F<0t6RpD*fQ@7D|g74=g}b zgbobA=2pDkszRb-1b3{8W9u@E;X;J^E)gq@{t6W)cw~a7nH3x+xjtm(-;oqpGDzueb8Zl*=U7y`YMunKUPB8WA|uiTqx{JtcBG z{*8rfl)}AV_TB)-DLkk|epA9^Nt^^_AXEb1l|aoa`UtjS00pNhgB2=u_a#{CGFw;~ z94kvBM5XLEC02XI;n*k_4>41yWPL;8be6+7d!3->8cifD5O@LH4CJ()!(|8cMlS4ia0~9eMX4#=ns7J~S1?ftwpYS*o)Lal0YoL*K?$&_ zuhC%|P5P*WD}%P+W-1=T1!RROD)aVBOxw%syj(#<<>1{CVh1+(lvd+|O2{%~H%&$) zszwNvnTI8oy)A)T`*>JPX9^px)7`(OsQq2RSGWIrar5P?8$bB)*^hkRI_EU%oKBsy zS?6rkIooy4PMxz`=j_!vPjYAFa9QgWW76x4ZHwXQ@5RWpEe54+F&=G;p=eu-Lfc{h z+7{f6?eTF=+i(bmhC@&^9D<|a5F`zUU}-o6O}WtJJ<27{_$*+BUm%Ll0!(~1al}xC zUkuZ%8e$lxM~uVth#pOk=-2d!-c66-V0r`-(B16P Date: Tue, 10 Nov 2026 14:21:31 +0000 Subject: [PATCH 08/93] Encode prompt states the id form; recall line states the real chrF floor; probe test over data/original Five of six encoders in the first original-corpus round wrote bare integer ids and were refused at parse (encoder-prompt-concept-ids). The prompt now says ids are written in full with their family prefix. Soul s1 on the original corpus: unrelated English scores chrF 13 to 18, not near 0, so the recall line says so; a test runs the probe over the real original items. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 2 +- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/context.rs | 18 ++++++++++ crates/weksa/tests/original_corpus.rs | 47 +++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index da6262d..3c73b0c 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -59,7 +59,7 @@ Fields: - predications: a list of {id, concept, frame, roles, features}; roles is a list of {role, filler}, filler a node id. - relations: a list of {relation, from, to, features}. - stance: {features}; the speaker's felt affect and restraint, never the affect the line displays. -Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. An entity comes only from the entities line. +Node ids are n1, n2, and so on, one per referent and predication. Every other id comes from the inventory listed below and must be live at its revision. Write each id in full, exactly as the inventory lines spell it, family prefix included: concept:7, never 7. An entity comes only from the entities line. A feature is {feature, value}; value is {\"Grade\": 0 to 1000}, {\"Option\": label} or {\"Flag\": true or false}, as the inventory lists the feature's kind."; /// The instructions a lowering prompt starts with: the lowerer's whole brief. diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 0d8f1d7..77c3642 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (completion from the opening; a high chrf means round scores on these items are recitation; unrelated words score near 0 and an in-voice guess about 20)", + "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 13 to 18, so a reading near that floor is no recitation and one well above it is)", self.n, self.answered, self.chrf ) } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index fe351d7..04ebe35 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -686,3 +686,21 @@ mod lowering { } } } + +/// First real rounds: five of six encoders wrote bare integers (`"concept": 7`) +/// where the inventory lists `concept:7`, and the draft reader refused them. +/// The prompt says the form outright, and the inventory lines it points to +/// are spelled in that form. +#[test] +fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { + assert!(ENCODE.contains("family prefix included: concept:7, never 7")); + let (text, _) = encode_with_prior("Hello there.", "Hi."); + let listed = text + .lines() + .skip_while(|l| !l.starts_with("inventory rev:")) + .skip(1); + for line in listed { + let id = line.split(": ").next().unwrap(); + assert!(id.contains(':'), "an inventory line spells its id in full: {line}"); + } +} diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 4416a36..51e4c35 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -65,3 +65,50 @@ fn the_original_inventory_is_the_dao_revision_three() { std::fs::copy(&own, ©).unwrap(); assert!(InventoryStore::open(©).is_ok()); } + +/// The probe over the real original corpus: every item is offered, none names a +/// work, and a model that answers with unrelated words reads at the noise floor. +#[test] +fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { + use weksa::model::{ModelPort, PortError}; + use weksa::run::{RecallOutcome, RunPaths, RunStores}; + struct Unrelated(Vec); + impl ModelPort for Unrelated { + fn model(&self) -> &str { + "unrelated" + } + fn answer(&mut self, p: &weksa::context::Prompt) -> Result { + self.0.push(p.text().to_string()); + Ok(r#"{"text":"stone harbours remember distant lanterns"}"#.to_string()) + } + } + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap(); + let path = |f: &str| { + std::fs::copy(src.join(f), tmp.path().join(f)).unwrap(); + tmp.path().join(f) + }; + let (c, i, e) = (path("corpus.cc"), path("inventory.cc"), path("entity.cc")); + let run = RunStores::create( + &tmp.path().join("run"), + "orig", + RunPaths { + corpus: &c, + inventory: &i, + entities: &e, + }, + "m", + "bag", + weksa::run::LowererReach::BriefedSubagent, + ) + .unwrap(); + let mut model = Unrelated(vec![]); + let RecallOutcome::Read(r) = run.recall(&mut model).unwrap() else { + panic!("answered"); + }; + assert_eq!((r.n, r.answered), (6, 6)); + assert!(r.chrf < 25.0, "{}", r.chrf); + assert_eq!(model.0.len(), 6); + assert!(model.0.iter().all(|t| t.contains("work: not published"))); + assert!(model.0.iter().all(|t| !t.contains("translator:"))); +} From 5ebdb3cff44512a14490b0198228e442a6a6abd3 Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:24:15 +0000 Subject: [PATCH 09/93] Soul s2 lows: the id-form test lists a real inventory; the floor reads 14 to 20 Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/run.rs | 2 +- crates/weksa/tests/context.rs | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 77c3642..bcf0d00 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -819,7 +819,7 @@ pub struct RecallReading { impl RecallReading { pub fn line(&self) -> String { format!( - "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 13 to 18, so a reading near that floor is no recitation and one well above it is)", + "recall n={} answered={} chrf={:.2} (completion from the opening; unrelated English already scores chrf 14 to 20, so a reading near that floor is no recitation and one well above it is)", self.n, self.answered, self.chrf ) } diff --git a/crates/weksa/tests/context.rs b/crates/weksa/tests/context.rs index 04ebe35..c379f0e 100644 --- a/crates/weksa/tests/context.rs +++ b/crates/weksa/tests/context.rs @@ -694,11 +694,22 @@ mod lowering { #[test] fn the_encode_prompt_states_the_id_form_the_inventory_lines_use() { assert!(ENCODE.contains("family prefix included: concept:7, never 7")); - let (text, _) = encode_with_prior("Hello there.", "Hi."); - let listed = text + let w = world(); + let run = run_over(w.dir.path()); + let inv = run.inventory().inventory().unwrap(); + let t = run.train().unwrap(); + let offer = run.offer("item:nibu:1").unwrap().unwrap(); + let text = Prompt::encode(&offer, &t, &inv, 4).text().to_string(); + let listed: Vec<&str> = text + .split_once("inventory rev:4\n") + .unwrap() + .1 + .split("\n\n") + .next() + .unwrap() .lines() - .skip_while(|l| !l.starts_with("inventory rev:")) - .skip(1); + .collect(); + assert!(!listed.is_empty(), "the prompt lists entries"); for line in listed { let id = line.split(": ").next().unwrap(); assert!(id.contains(':'), "an inventory line spells its id in full: {line}"); From 2259b9f8c3fc4bd48deb4ca13a4b9cd9b822b66f Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 14:26:14 +0000 Subject: [PATCH 10/93] WIP (does not build): DraftClass on DraftError; create() call sites take a ModelFamily argument Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/encode.rs | 62 +++++++++++++++++++++++++++ crates/weksa/src/run.rs | 2 +- crates/weksa/tests/golf_loop.rs | 28 ++++++------ crates/weksa/tests/original_corpus.rs | 2 +- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 5071316..2efb331 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -47,6 +47,67 @@ pub struct PacketDraft { pub struct DraftError { pub line: usize, pub column: usize, + /// What kind of reading failure, from serde's own fixed vocabulary. It + /// never holds a token of the answer. + pub class: DraftClass, +} + +/// Why an answer is not a draft: a closed set of classes, never the offending +/// text, so a record may keep it for a held-out item. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DraftClass { + /// Not JSON. + Syntax, + /// JSON that ends early. + Eof, + MissingField, + UnknownField, + /// A value of the wrong JSON type, such as a bare integer for an id. + InvalidType, + /// A value of the right type that is not one of the field's: a node id + /// spelled `entity:x`, a number out of range. + InvalidValue, + Other, +} + +impl DraftClass { + /// The code a round record keeps. + pub fn code(self) -> &'static str { + match self { + DraftClass::Syntax => "syntax", + DraftClass::Eof => "eof", + DraftClass::MissingField => "missing-field", + DraftClass::UnknownField => "unknown-field", + DraftClass::InvalidType => "invalid-type", + DraftClass::InvalidValue => "invalid-value", + DraftClass::Other => "other", + } + } + + fn of(e: &serde_json::Error) -> Self { + use serde_json::error::Category; + match e.classify() { + Category::Syntax => DraftClass::Syntax, + Category::Eof => DraftClass::Eof, + Category::Io => DraftClass::Other, + // Only the leading words of serde's message are read, and only to + // pick one of these classes; the message is not kept. + Category::Data => { + let msg = e.to_string(); + if msg.starts_with("missing field") { + DraftClass::MissingField + } else if msg.starts_with("unknown field") { + DraftClass::UnknownField + } else if msg.starts_with("invalid type") { + DraftClass::InvalidType + } else if msg.starts_with("invalid value") { + DraftClass::InvalidValue + } else { + DraftClass::Other + } + } + } + } } impl fmt::Display for DraftError { @@ -72,6 +133,7 @@ fn from_json(answer: &str) -> Result { serde_json::from_str(answer.trim()).map_err(|e| DraftError { line: e.line(), column: e.column(), + class: DraftClass::of(&e), }) } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index bcf0d00..517101d 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -2059,7 +2059,7 @@ mod tests { entities: &entities, }, "fake-model", - "bag", crate::run::LowererReach::ToolLessRelay, + "bag", crate::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); let mut model = fake(); diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 4d9dc1c..377e8f3 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -47,7 +47,7 @@ fn a_run_pins_its_stores_and_refuses_drift() { entities: &e, }, model, - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; @@ -148,7 +148,7 @@ fn a_run_works_on_its_own_copy_of_the_inventory() { entities: &e, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); @@ -223,7 +223,7 @@ fn create_refuses_a_root_that_holds_an_inventory_and_never_truncates_it() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; @@ -268,7 +268,7 @@ fn a_create_that_fails_leaves_no_inventory_and_the_retry_succeeds() { entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) }; // A directory opens for reading and fails mid-copy, after the destination exists. @@ -381,14 +381,14 @@ fn a_store_changed_between_its_digest_and_its_open_is_drift() { entities: &e, }; assert_eq!( - RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || more(&mut w)).err(), + RunStores::create_between(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || more(&mut w)).err(), Some(RunError::Drift(DriftOf::Corpus)) ); assert!(!root.join("inventory.cc").exists()); // Not the copy and not the lock the store's open left beside it. assert!(!root.join("inventory.cc.lock").exists()); assert_eq!(RunStores::open(&root).err(), Some(RunError::NoRun)); - RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay).unwrap(); + RunStores::create(&root, "t", paths(), "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded).unwrap(); } /// Runs `change` in the window between the pins and the open, at `open` and at @@ -411,7 +411,7 @@ fn window( inventory: &i, entities: &e, }; - let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, || { + let created = RunStores::create_between(&root, "t", paths, "fake-model", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded, || { change(&mut w, &root) }) .err(); @@ -828,7 +828,7 @@ fn dao_over(edit: impl FnOnce(&Path, &Path)) -> Dao { entities: &entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); Dao { @@ -1477,15 +1477,15 @@ fn errors_echo_no_input() { inventory: &missing, entities: &missing, }; - let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, "ok", stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); - let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, &format!("{canary}!"), stores, "m", "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); - let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay) + let e = RunStores::create(&root, "ok", stores, &format!("{canary}\n"), "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded) .err() .unwrap(); note(&e, &e); @@ -1519,7 +1519,7 @@ fn errors_echo_no_input() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); let mut model = cycle_fake(&format!("{{\"{canary}\":1}}")); @@ -2449,7 +2449,7 @@ fn a_changed_head_is_drift() { entities: &dao.entities, }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, ModelFamily::Unrecorded ) .unwrap(); assert_eq!( @@ -2768,7 +2768,7 @@ fn a_summary_line_says_how_lowerings_were_answered() { inventory: &dao.inventory, entities: &dao.entities, }; - RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent) + RunStores::create(&root, "b", paths, "fake-model", "bag", LowererReach::BriefedSubagent, ModelFamily::Unrecorded) .unwrap(); let run = RunStores::open(&root).unwrap(); assert_eq!(run.record().lowerer_reach, LowererReach::BriefedSubagent); diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index 51e4c35..bbf7056 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -99,7 +99,7 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { }, "m", "bag", - weksa::run::LowererReach::BriefedSubagent, + weksa::run::LowererReach::BriefedSubagent, ModelFamily::Unrecorded ) .unwrap(); let mut model = Unrelated(vec![]); From 2759c0f12d651aa2890b84c63304ab164b28d67e Mon Sep 17 00:00:00 2001 From: Weksa Date: Tue, 10 Nov 2026 17:05:07 +0000 Subject: [PATCH 11/93] Round records carry a closed reason for every parse refusal and an own-family count DraftClass (closed set from serde's category and leading words, never the message) rides on DraftError; the round counts parse_reasons by code ('unreadable' for an answer that did not read as text) beside admit_rules. RunRecord keeps the answerer's ModelFamily (loop open --answerer-family); the round counts offered ModelWritten items of that family and the line says own-family N. The corpus rule is documented as binding model judges: the loop labels, it does not forbid. New fields and admit_rules are cultcache default, so older run.cc records decode; schemas regenerated. Co-Authored-By: Claude Sonnet 5.5 --- crates/weksa/src/context.rs | 12 +- crates/weksa/src/corpus.rs | 8 +- crates/weksa/src/encode.rs | 35 ++- crates/weksa/src/main.rs | 6 + crates/weksa/src/run.rs | 88 +++++- crates/weksa/tests/cli.rs | 4 +- crates/weksa/tests/common/mod.rs | 2 +- crates/weksa/tests/golf_loop.rs | 277 ++++++++++++++++++ crates/weksa/tests/original_corpus.rs | 76 +++++ schemas/cultnet/weksa.corpus_item.schema.json | 2 +- schemas/cultnet/weksa.loop_round.schema.json | 17 +- schemas/cultnet/weksa.loop_run.schema.json | 26 ++ 12 files changed, 528 insertions(+), 25 deletions(-) diff --git a/crates/weksa/src/context.rs b/crates/weksa/src/context.rs index 3c73b0c..5592e1e 100644 --- a/crates/weksa/src/context.rs +++ b/crates/weksa/src/context.rs @@ -86,7 +86,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// assert!(t.is_empty()); /// ``` @@ -100,7 +100,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let _u = t.clone(); /// ``` @@ -111,7 +111,7 @@ Answer with one JSON object and nothing else: {\"text\": \"...\"}, the continuat /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// s(&t); /// ``` @@ -317,7 +317,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -335,7 +335,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); @@ -348,7 +348,7 @@ impl EditFeedback { /// # let p = |n: &str| dir.path().join(n); /// # for n in ["c.cc", "i.cc", "e.cc"] { std::fs::write(p(n), [0x90]).unwrap(); } /// # let paths = weksa::run::RunPaths { corpus: &p("c.cc"), inventory: &p("i.cc"), entities: &p("e.cc") }; -/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay).unwrap(); +/// # let run = weksa::run::RunStores::create(&p("run"), "t", paths, "m", "e", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded).unwrap(); /// let t = run.train().unwrap(); /// let inv = run.inventory().inventory().unwrap(); /// let feedback = weksa::context::EditFeedback::none(); diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index 2604c6a..a2f94ae 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -43,7 +43,10 @@ pub enum Source { translator: String, edition_year: u16, }, - /// Labelled model output; never in the test split, never judged by its own family. + /// Labelled model output; never in the test split. The rule that it is + /// never judged by its own family binds model judges and the split; a + /// deterministic scorer refuses nothing, so the loop labels (the round + /// counts items of the answerer's family) and does not forbid. /// `family` is `Unrecorded` when nothing records one. `generator` is the /// string the fixture or run wrote, kept verbatim. ModelWritten { @@ -54,7 +57,7 @@ pub enum Source { } /// The family of the model that wrote a line. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum ModelFamily { Anthropic, OpenAi, @@ -63,6 +66,7 @@ pub enum ModelFamily { Meta, Mistral, /// Nothing records which family wrote the line. + #[default] Unrecorded, } diff --git a/crates/weksa/src/encode.rs b/crates/weksa/src/encode.rs index 2efb331..a71a0b9 100644 --- a/crates/weksa/src/encode.rs +++ b/crates/weksa/src/encode.rs @@ -64,9 +64,12 @@ pub enum DraftClass { UnknownField, /// A value of the wrong JSON type, such as a bare integer for an id. InvalidType, - /// A value of the right type that is not one of the field's: a node id - /// spelled `entity:x`, a number out of range. + /// A value of the right type that is not one of the field's, such as a + /// number out of range. InvalidValue, + /// A string that is not the form its field names: a node id spelled + /// `entity:x`, an id of the wrong kind ([`crate::form::FormError`]). + BadForm, Other, } @@ -80,6 +83,7 @@ impl DraftClass { DraftClass::UnknownField => "unknown-field", DraftClass::InvalidType => "invalid-type", DraftClass::InvalidValue => "invalid-value", + DraftClass::BadForm => "bad-form", DraftClass::Other => "other", } } @@ -102,6 +106,8 @@ impl DraftClass { DraftClass::InvalidType } else if msg.starts_with("invalid value") { DraftClass::InvalidValue + } else if msg.starts_with("not a ") { + DraftClass::BadForm } else { DraftClass::Other } @@ -312,6 +318,31 @@ mod tests { CorpusItemId::parse("item:daodejing:1").unwrap() } + #[test] + fn a_draft_error_is_classed_from_serdes_vocabulary_and_keeps_no_text() { + let class = |answer: &str| PacketDraft::parse(answer).unwrap_err().class; + assert_eq!(class("not json"), DraftClass::Syntax); + assert_eq!(class("{\"referents\": ["), DraftClass::Eof); + assert_eq!(class("{}"), DraftClass::MissingField); + // The classes serde's own messages open with, read off a plain error. + let of = |r: Result| DraftClass::of(&r.unwrap_err()); + assert_eq!(of(serde_json::from_str("300")), DraftClass::InvalidValue); + assert_eq!(of(serde_json::from_str("\"x\"")), DraftClass::InvalidType); + // Every code is distinct and none can hold answer text: they are fixed. + let all = [ + DraftClass::Syntax, + DraftClass::Eof, + DraftClass::MissingField, + DraftClass::UnknownField, + DraftClass::InvalidType, + DraftClass::InvalidValue, + DraftClass::BadForm, + DraftClass::Other, + ]; + let codes: std::collections::BTreeSet<_> = all.iter().map(|c| c.code()).collect(); + assert_eq!(codes.len(), all.len()); + } + #[test] fn ids_are_pinned_digests_of_their_inputs() { let encoder = encoder_id("fake-model", "abcd"); diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 3808672..5b88c14 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -92,6 +92,10 @@ enum LoopCommand { /// rounds only) or tool-less-relay. Pinned in the run's record. #[arg(long)] lowerer_reach: LowererReach, + /// The family of the answering model; rounds count the corpus items + /// that family wrote. Defaults to unrecorded. + #[arg(long, default_value = "unrecorded")] + answerer_family: ModelFamilyArg, }, /// Run one round of the run at ROOT. The stores are read only from the /// run's record: the global --corpus, --inventory and --entities are not @@ -744,6 +748,7 @@ fn main() -> Result<()> { answerer, embed_model, lowerer_reach, + answerer_family, }) => { let run = RunStores::create( &root, @@ -756,6 +761,7 @@ fn main() -> Result<()> { &answerer, &embed_model, lowerer_reach, + answerer_family.into(), )?; println!("opened run:{}", run.record().label); } diff --git a/crates/weksa/src/run.rs b/crates/weksa/src/run.rs index 517101d..6d0195f 100644 --- a/crates/weksa/src/run.rs +++ b/crates/weksa/src/run.rs @@ -35,7 +35,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::context::{EditFeedback, EncoderOffer, Prompt, TrainItems}; -use crate::corpus::{CorpusItem, CorpusItemId, Source}; +use crate::corpus::{CorpusItem, CorpusItemId, ModelFamily, Source}; use crate::encode::{EditDraft, LoweringDraft, PacketDraft, encoder_id}; use crate::form::{is_attestation, is_slug}; use crate::model::{ModelPort, PortError}; @@ -107,6 +107,12 @@ pub struct RunRecord { /// How the run's lowering prompts are answered, set once at open. #[cultcache(key = 12)] pub lowerer_reach: LowererReach, + /// The family of the model answering the run's encode and lowering + /// prompts, set once at open; `Unrecorded` when the opener did not say + /// and for a run opened before this was kept. + #[cultcache(key = 13, default)] + #[schemars(default)] + pub answerer_family: ModelFamily, } /// How a run's lowering prompts reach the model that answers them (rulings @@ -224,6 +230,8 @@ pub struct RefusalCounts { /// inventory_digest: String::new(), /// cycle: None, /// admit_rules: vec![], +/// parse_reasons: vec![], +/// own_family: 0, /// } /// } /// ``` @@ -276,9 +284,22 @@ pub struct RoundRecord { cycle: Option, /// The rules admission refused items under, in rule order; every Train /// and Validation item refused at `Stage::Admit` is counted once per rule. - #[cultcache(key = 18)] + #[cultcache(key = 18, default)] #[schemars(default)] admit_rules: Vec, + /// Why items were refused at `Stage::Parse`, in code order, over Train and + /// Validation: a [`DraftClass`] code (`rule`), or `unreadable` for an + /// item with no answer that read as text. Never a token of an answer. + #[cultcache(key = 19, default)] + #[schemars(default)] + parse_reasons: Vec, + /// Offered (live Train and Validation) `ModelWritten` items whose writer + /// family is the run's answerer family. The corpus rule binds + /// model judges; the scorer is deterministic, so nothing is refused for + /// this: the round labels it. + #[cultcache(key = 20, default)] + #[schemars(default)] + own_family: u32, } impl RoundRecord { @@ -336,6 +357,12 @@ impl RoundRecord { pub fn admit_rules(&self) -> &[RuleCount] { &self.admit_rules } + pub fn parse_reasons(&self) -> &[RuleCount] { + &self.parse_reasons + } + pub fn own_family(&self) -> u32 { + self.own_family + } /// The round line, unlabelled: [`RunStores::round_line`] adds how the run was lowered. `n` is every item of the split, /// `scored` those that lowered; a mean counts a refused item as 0. @@ -369,16 +396,24 @@ impl RoundRecord { x.admit, x.lower, ); - if self.admit_rules.is_empty() { - line - } else { - let rules: Vec = self - .admit_rules + let codes = |counts: &[RuleCount]| { + counts .iter() .map(|r| format!("{}x{}", r.rule, r.count)) - .collect(); - format!("{line} admit-rules={}", rules.join(",")) + .collect::>() + .join(",") + }; + let mut line = line; + if !self.parse_reasons.is_empty() { + line = format!("{line} parse-reasons={}", codes(&self.parse_reasons)); } + if !self.admit_rules.is_empty() { + line = format!("{line} admit-rules={}", codes(&self.admit_rules)); + } + if self.own_family > 0 { + line = format!("{line} own-family {}", self.own_family); + } + line } } @@ -550,13 +585,15 @@ impl RunStores { model: &str, embedder: &str, reach: LowererReach, + answerer_family: ModelFamily, ) -> Result { - Self::create_between(root, label, paths, model, embedder, reach, || {}) + Self::create_between(root, label, paths, model, embedder, reach, answerer_family, || {}) } /// `create` with `between` run after the digests are taken and before the /// stores are opened: the seam that lets a test change a store in that window. #[doc(hidden)] + #[allow(clippy::too_many_arguments)] // `create`'s six plus the test seam pub fn create_between( root: &Path, label: &str, @@ -564,6 +601,7 @@ impl RunStores { model: &str, embedder: &str, reach: LowererReach, + answerer_family: ModelFamily, between: impl FnOnce(), ) -> Result { if !is_slug(label) { @@ -592,6 +630,7 @@ impl RunStores { chrf_version: CHRF_VERSION.to_string(), inventory_digest: String::new(), lowerer_reach: reach, + answerer_family, }; // The run works on its own copy; the caller's store is never opened again. let head = root.join(INVENTORY_FILE); @@ -1024,13 +1063,21 @@ fn score_round( let mut items: Vec = Vec::new(); let mut fates: Vec = Vec::new(); let mut admit_rules: BTreeMap = BTreeMap::new(); + let mut parse_reasons: BTreeMap = BTreeMap::new(); // Phase 1 ends at admission: each admitted view waits for its lowering. let mut admitted: Vec<(usize, CorpusItemId, String, LoweringInput)> = Vec::new(); for (key, answer) in &answers { let item = run.corpus.get(key).map_err(store)?.ok_or(RunError::Store)?; let id = CorpusItemId::parse(key).map_err(|_| RunError::Store)?; let fate = match answer.as_deref().map(PacketDraft::parse) { - None | Some(Err(_)) => Fate::Refused(Stage::Parse, None), + None => { + *parse_reasons.entry("unreadable".to_string()).or_default() += 1; + Fate::Refused(Stage::Parse, None) + } + Some(Err(e)) => { + *parse_reasons.entry(e.class.code().to_string()).or_default() += 1; + Fate::Refused(Stage::Parse, None) + } Some(Ok(draft)) => { let packet = draft.into_packet(&id, rev, &encoder); let packet_key = packet.id.to_string(); @@ -1172,6 +1219,15 @@ fn score_round( } } + let own_family = match run.record.answerer_family { + ModelFamily::Unrecorded => 0, + family => items + .iter() + .filter(|item| { + matches!(&item.source, Source::ModelWritten { family: f, .. } if *f == family) + }) + .count() as u32, + }; let n = run.rounds().map_err(store)?.len() as u32 + 1; let record = RoundRecord { n, @@ -1195,6 +1251,11 @@ fn score_round( .into_iter() .map(|(rule, count)| RuleCount { rule, count }) .collect(), + parse_reasons: parse_reasons + .into_iter() + .map(|(rule, count)| RuleCount { rule, count }) + .collect(), + own_family, }; Ok(RoundOutcome::Recorded(record)) } @@ -1628,6 +1689,8 @@ mod tests { inventory_digest: String::new(), cycle: None, admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, }; assert_eq!(r.n(), 1); } @@ -1844,6 +1907,8 @@ mod tests { inventory_digest: digest.to_string(), cycle: None, admit_rules: vec![], + parse_reasons: vec![], + own_family: 0, } } @@ -1861,6 +1926,7 @@ mod tests { chrf_version: String::new(), inventory_digest: digest.to_string(), lowerer_reach: LowererReach::ToolLessRelay, + answerer_family: ModelFamily::Unrecorded, } } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index be64338..56a32af 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -1346,6 +1346,8 @@ fn loop_open_and_unanswered_round() { "fake-model", "--lowerer-reach", "tool-less-relay", + "--answerer-family", + "open-ai", ], ) }; @@ -1395,7 +1397,7 @@ fn loop_open_and_unanswered_round() { assert!(second.status.success(), "{}", out(&second)); assert_eq!( out(&second), - "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 lowering=tool-less-relay\n" + "round 1 rev 3 train n=1 scored=0 chrf=0.00 meaning=0.0000 refused=1 validation n=2 scored=0 chrf=0.00 meaning=0.0000 refused parse=2 admit=0 lower=0 parse-reasons=syntaxx3 lowering=tool-less-relay\n" ); // The global store flags are not read by loop round: a drifted store is refused. diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 189202b..7bd9338 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -131,7 +131,7 @@ pub fn run_over(dir: &std::path::Path) -> weksa::run::RunStores { entities: &dir.join("entities.cc"), }, "fake-model", - "bag", weksa::run::LowererReach::ToolLessRelay, + "bag", weksa::run::LowererReach::ToolLessRelay, weksa::corpus::ModelFamily::Unrecorded, ) .unwrap() } diff --git a/crates/weksa/tests/golf_loop.rs b/crates/weksa/tests/golf_loop.rs index 377e8f3..60abc38 100644 --- a/crates/weksa/tests/golf_loop.rs +++ b/crates/weksa/tests/golf_loop.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use common::{ent, run_over, world}; use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; +use weksa::corpus::ModelFamily; use weksa::run::{DriftOf, RunError, RunPaths, RunRecord, RunStores}; use weksa::split::fnv1a64; @@ -2947,6 +2948,282 @@ fn admit_refusals_are_counted_by_rule_and_printed_with_the_label() { assert!(!dao.run.round_line(&r).contains("admit-rules")); } +/// Item 1's good draft as JSON, changed by `change`, as the model's answer. +fn mangled(change: impl FnOnce(&mut serde_json::Value)) -> String { + let mut v: serde_json::Value = serde_json::from_str(&json(&draft_of(1))).unwrap(); + change(&mut v); + v.to_string() +} + +/// The round record of a Dao run whose item 1 is answered with `answer` and +/// the rest well. +fn round_with_item_1(answer: String) -> (Dao, RoundRecord) { + let mut dao = dao(); + let mut answers = good_answers(); + answers[0].1 = answer; + let mut model = fake(&answers); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + (dao, r) +} + +fn reasons(r: &RoundRecord) -> Vec<(&str, u32)> { + r.parse_reasons() + .iter() + .map(|c| (c.rule.as_str(), c.count)) + .collect() +} + +#[test] +fn parse_reasons_by_class() { + let good = json(&draft_of(1)); + let truncated = good[..good.len() / 2].to_string(); + let cases: Vec<(&str, String)> = vec![ + ("syntax", "not json at all".to_string()), + ("eof", truncated), + ( + "invalid-type", + mangled(|v| v["referents"][0]["concept"] = serde_json::json!(7)), + ), + ( + "bad-form", + mangled(|v| v["referents"][0]["concept"] = serde_json::json!("entity:x")), + ), + ( + "unknown-field", + mangled(|v| v["surprise"] = serde_json::json!(1)), + ), + ( + "missing-field", + mangled(|v| { + v.as_object_mut().unwrap().remove("referents"); + }), + ), + ]; + for (code, answer) in cases { + let (dao, r) = round_with_item_1(answer); + assert_eq!(reasons(&r), [(code, 1)], "{code}"); + assert_eq!(r.train()[0].outcome, Outcome::Refused(Stage::Parse), "{code}"); + let line = dao.run.round_line(&r); + assert!(line.contains(&format!("parse-reasons={code}x1")), "{line}"); + } + // A round nothing refused at Parse carries and prints none. + let (dao, r) = round_with_item_1(good); + assert!(r.parse_reasons().is_empty()); + assert!(!dao.run.round_line(&r).contains("parse-reasons")); +} + +#[test] +fn parse_reasons_count_per_code_over_both_splits_and_name_unreadable() { + let mut dao = dao(); + let expected = prompts(&dao.run); + let answers = keyed(&good_answers()); + let round1 = dao.root.join("round-1"); + std::fs::create_dir_all(&round1).unwrap(); + for (key, p) in &expected { + let bytes = match key.as_str() { + // Train: not UTF-8, so no answer parsed from the file. + "item:daodejing:1" => vec![0xff, 0xfe], + // Validation: not JSON. + "item:daodejing:2" => b"nope".to_vec(), + _ => answers[key].clone().into_bytes(), + }; + std::fs::write(round1.join(format!("{}.answer", p.digest())), bytes).unwrap(); + } + let table = table_english(); + for (key, p) in lowering_prompts(&dao, dao.run.inventory()) + .iter() + .filter(|(key, _)| key == "item:daodejing:3") + { + let answer = serde_json::json!({ "text": table[key] }).to_string(); + std::fs::write(round1.join(format!("{}.answer", p.digest())), answer).unwrap(); + } + let mut files = AnswerFiles::new(round1, "fake-model".into()); + let r = recorded(round(&mut dao.run, &mut files, &Scorer::new(Bag)).unwrap()); + assert_eq!(reasons(&r), [("syntax", 1), ("unreadable", 1)]); +} + +#[test] +fn parse_reasons_never_echo_answer() { + let canary = "CANARY-7f3a-held-out"; + // The canary sits where serde would quote it: an unknown field name, an + // invalid value, and the raw text of a non-JSON answer. + let answers = [ + mangled(|v| v[canary] = serde_json::json!(1)), + mangled(|v| v["referents"][0]["concept"] = serde_json::json!(canary)), + format!("{canary} is not json"), + ]; + for answer in answers { + // Item 2 is Validation: the held-out item carries the canary. + let mut dao = dao(); + let mut a = good_answers(); + a[1].1 = answer; + let mut model = fake(&a); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + assert_eq!(r.validation_refused().parse, 1); + let line = dao.run.round_line(&r); + assert!(!line.contains(canary) && !line.contains("CANARY"), "{line}"); + let bytes = std::fs::read(dao.root.join("run.cc")).unwrap(); + assert!(!bytes.windows(6).any(|w| w == b"CANARY")); + assert!(!format!("{r:?}").contains("CANARY")); + } +} + +/// A run record as written before `answerer_family` (key 13) existed. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_run")] +struct PreFamilyRun { + #[cultcache(key = 0)] + label: String, + #[cultcache(key = 1)] + corpus: String, + #[cultcache(key = 2)] + inventory: String, + #[cultcache(key = 3)] + entities: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + prompt_digest: String, + #[cultcache(key = 8)] + model: String, + #[cultcache(key = 9)] + embedder: String, + #[cultcache(key = 10)] + chrf_version: String, + #[cultcache(key = 11)] + inventory_digest: String, + #[cultcache(key = 12)] + lowerer_reach: weksa::run::LowererReach, +} + +/// A round record as written before `parse_reasons` and `own_family`, and +/// before `admit_rules` had a default. +#[derive(Clone, Debug, schemars::JsonSchema, cultcache_rs::DatabaseEntry)] +#[cultcache(type = "weksa.loop_round")] +struct LegacyRound { + #[cultcache(key = 0)] + n: u32, + #[cultcache(key = 1)] + inventory_rev: u32, + #[cultcache(key = 2)] + prompt_digest: String, + #[cultcache(key = 4)] + corpus_digest: String, + #[cultcache(key = 5)] + entity_digest: String, + #[cultcache(key = 6)] + model: String, + #[cultcache(key = 7)] + embedder: String, + #[cultcache(key = 8)] + chrf_version: String, + #[cultcache(key = 9)] + calls: Vec, + #[cultcache(key = 10)] + train: Vec, + #[cultcache(key = 11)] + train_mean: weksa::score::SplitMean, + #[cultcache(key = 12)] + validation_mean: weksa::score::SplitMean, + #[cultcache(key = 13)] + validation_refused: weksa::run::RefusalCounts, + #[cultcache(key = 14)] + train_scored: u32, + #[cultcache(key = 15)] + validation_scored: u32, + #[cultcache(key = 16)] + inventory_digest: String, + #[cultcache(key = 17)] + cycle: Option, +} + +#[test] +fn records_from_before_the_new_fields_decode_with_defaults() { + let mut dao = dao(); + let mut model = fake(&good_answers()); + let r = recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap()); + let rec = dao.run.record().clone(); + let root = dao.root.clone(); + let legacy = LegacyRound { + n: r.n(), + inventory_rev: r.inventory_rev(), + prompt_digest: r.prompt_digest().into(), + corpus_digest: r.corpus_digest().into(), + entity_digest: r.entity_digest().into(), + model: r.model().into(), + embedder: r.embedder().into(), + chrf_version: r.chrf_version().into(), + calls: r.calls().to_vec(), + train: r.train().to_vec(), + train_mean: r.train_mean().clone(), + validation_mean: r.validation_mean().clone(), + validation_refused: r.validation_refused(), + train_scored: r.train_scored(), + validation_scored: r.validation_scored(), + inventory_digest: r.inventory_digest().into(), + cycle: r.cycle(), + }; + let run = PreFamilyRun { + label: rec.label.clone(), + corpus: rec.corpus.clone(), + inventory: rec.inventory.clone(), + entities: rec.entities.clone(), + corpus_digest: rec.corpus_digest.clone(), + entity_digest: rec.entity_digest.clone(), + prompt_digest: rec.prompt_digest.clone(), + model: rec.model.clone(), + embedder: rec.embedder.clone(), + chrf_version: rec.chrf_version.clone(), + inventory_digest: rec.inventory_digest.clone(), + lowerer_reach: rec.lowerer_reach, + }; + drop(dao.run); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache + .add_generic_backing_store(SingleFileMessagePackBackingStore::new(root.join("run.cc"))) + .unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("run:dao".to_string(), &run).unwrap(); + cache.put("run:dao:round:1".to_string(), &legacy).unwrap(); + drop(cache); + + let reopened = RunStores::open(&root).unwrap(); + assert_eq!(reopened.record().answerer_family, ModelFamily::Unrecorded); + let rounds = reopened.rounds().unwrap(); + assert_eq!(rounds.len(), 1); + assert_eq!(rounds[0].n(), 1); + assert!(rounds[0].admit_rules().is_empty()); + assert!(rounds[0].parse_reasons().is_empty()); + assert_eq!(rounds[0].own_family(), 0); +} + +#[test] +fn the_published_run_and_round_schemas_require_none_of_the_new_fields() { + let published = weksa::schema::published(); + let schema = |needle: &str| -> serde_json::Value { + let (_, json) = published + .iter() + .find(|(name, _)| name.contains(needle)) + .unwrap(); + serde_json::from_str(json).unwrap() + }; + for (needle, fields) in [ + ("loop_round", &["parse_reasons", "own_family"][..]), + ("loop_run", &["answerer_family"][..]), + ] { + let s = schema(needle); + let required = s["required"].as_array().unwrap(); + for f in fields { + assert!(s["properties"][f].is_object(), "{needle} {f}"); + assert!(!required.iter().any(|r| r == f), "{needle} {f} is required"); + } + } +} + /// Answers a recall probe from the corpus: the item's own second half, a /// fixed wrong text, or junk. struct Reciter { diff --git a/crates/weksa/tests/original_corpus.rs b/crates/weksa/tests/original_corpus.rs index bbf7056..cffd4f2 100644 --- a/crates/weksa/tests/original_corpus.rs +++ b/crates/weksa/tests/original_corpus.rs @@ -112,3 +112,79 @@ fn the_recall_probe_covers_all_six_original_items_and_names_no_work() { assert!(model.0.iter().all(|t| t.contains("work: not published"))); assert!(model.0.iter().all(|t| !t.contains("translator:"))); } + +/// Answers every prompt with text that is not a draft: nothing is admitted, so +/// the round measures only what is counted before admission. +struct Nope; + +impl weksa::model::ModelPort for Nope { + fn model(&self) -> &str { + "nope" + } + fn answer( + &mut self, + _: &weksa::context::Prompt, + ) -> Result { + Ok("not a draft".to_string()) + } +} + +struct Bag; + +impl weksa::score::Embedder for Bag { + fn model(&self) -> &str { + "bag" + } + fn embed(&self, texts: &[&str]) -> anyhow::Result>> { + Ok(texts.iter().map(|_| vec![1.0]).collect()) + } +} + +fn original_round(family: ModelFamily) -> (weksa::run::RunStores, weksa::run::RoundRecord) { + use weksa::run::{LowererReach, RoundOutcome, RunPaths, RunStores, round}; + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../data/original"); + let tmp = tempfile::tempdir().unwrap().keep(); + for f in ["corpus.cc", "entity.cc", "inventory.cc"] { + std::fs::copy(src.join(f), tmp.join(f)).unwrap(); + } + let mut run = RunStores::create( + &tmp.join("run"), + "orig", + RunPaths { + corpus: &tmp.join("corpus.cc"), + inventory: &tmp.join("inventory.cc"), + entities: &tmp.join("entity.cc"), + }, + "nope", + "bag", + LowererReach::ToolLessRelay, + family, + ) + .unwrap(); + let scorer = weksa::score::Scorer::new(Bag); + match round(&mut run, &mut Nope, &scorer).unwrap() { + RoundOutcome::Recorded(r) => (run, r), + RoundOutcome::Unanswered { .. } => panic!("answered"), + } +} + +#[test] +fn own_family_counts_and_labels() { + // The original corpus is six Anthropic-written items, all offered. + let (run, r) = original_round(ModelFamily::Anthropic); + assert_eq!(r.own_family(), 6); + assert_eq!( + r.parse_reasons().iter().map(|c| c.count).sum::(), + 6, + "every item was refused at Parse and says why" + ); + let line = run.round_line(&r); + assert!(line.contains(" own-family 6"), "{line}"); + + // A different answerer family, or none recorded, labels nothing. + for family in [ModelFamily::OpenAi, ModelFamily::Unrecorded] { + let (run, r) = original_round(family); + assert_eq!(r.own_family(), 0); + assert!(!run.round_line(&r).contains("own-family")); + } +} diff --git a/schemas/cultnet/weksa.corpus_item.schema.json b/schemas/cultnet/weksa.corpus_item.schema.json index c82c4da..b5e5bc9 100644 --- a/schemas/cultnet/weksa.corpus_item.schema.json +++ b/schemas/cultnet/weksa.corpus_item.schema.json @@ -218,7 +218,7 @@ ] }, { - "description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.", + "description": "Labelled model output; never in the test split. The rule that it is\nnever judged by its own family binds model judges and the split; a\ndeterministic scorer refuses nothing, so the loop labels (the round\ncounts items of the answerer's family) and does not forbid.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.", "type": "object", "properties": { "ModelWritten": { diff --git a/schemas/cultnet/weksa.loop_round.schema.json b/schemas/cultnet/weksa.loop_round.schema.json index f218bdb..4117573 100644 --- a/schemas/cultnet/weksa.loop_round.schema.json +++ b/schemas/cultnet/weksa.loop_round.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "RoundRecord", - "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n }\n}\n```", + "description": "One scored round. It holds no text and no Test reading: per-item rows for\nTrain only, Validation only as means and refusal counts by stage. Only\n`run::round` builds one.\n\n```\nfn f(r: &weksa::run::RoundRecord) -> u32 { r.n() }\n```\n\nEvery field is given, so the literal below fails only on privacy; its twin\nis the same literal in the unit test `a_round_record_literal_compiles_in_the_lib`.\n\n```compile_fail,E0451\nuse weksa::run::{RefusalCounts, RoundRecord};\nuse weksa::score::SplitMean;\nfn f() -> RoundRecord {\n RoundRecord {\n n: 1,\n inventory_rev: 1,\n prompt_digest: String::new(),\n corpus_digest: String::new(),\n entity_digest: String::new(),\n model: String::new(),\n embedder: String::new(),\n chrf_version: String::new(),\n calls: vec![],\n train: vec![],\n train_mean: SplitMean::default(),\n train_scored: 0,\n validation_mean: SplitMean::default(),\n validation_scored: 0,\n validation_refused: RefusalCounts::default(),\n inventory_digest: String::new(),\n cycle: None,\n admit_rules: vec![],\n parse_reasons: vec![],\n own_family: 0,\n }\n}\n```", "type": "object", "properties": { "admit_rules": { @@ -57,6 +57,21 @@ "format": "uint32", "minimum": 0 }, + "own_family": { + "description": "Offered (live Train and Validation) `ModelWritten` items whose writer\nfamily is the run's answerer family. The corpus rule binds\nmodel judges; the scorer is deterministic, so nothing is refused for\nthis: the round labels it.", + "type": "integer", + "format": "uint32", + "default": 0, + "minimum": 0 + }, + "parse_reasons": { + "description": "Why items were refused at `Stage::Parse`, in code order, over Train and\nValidation: a [`DraftClass`] code (`rule`), or `unreadable` for an\nitem with no answer that read as text. Never a token of an answer.", + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/RuleCount" + } + }, "prompt_digest": { "type": "string" }, diff --git a/schemas/cultnet/weksa.loop_run.schema.json b/schemas/cultnet/weksa.loop_run.schema.json index b5b5ab8..b339def 100644 --- a/schemas/cultnet/weksa.loop_run.schema.json +++ b/schemas/cultnet/weksa.loop_run.schema.json @@ -4,6 +4,11 @@ "description": "What a run pinned at open. Key `run: