scene-in-group: rubric r1 committed; all 98 keeper scenes re-annotated by it #77
7 changed files with 349 additions and 5 deletions
|
|
@ -5,8 +5,8 @@ culture, and constraint, and the planned lowerer that turns an agent's
|
|||
meaning packet into the reader's display language.
|
||||
|
||||
**What is valuable here.** The interlingua standard and the lowering design: a
|
||||
meaning packet owns what is meant, the scene state (live: `weksa scene put|dump`; speaker
|
||||
documents are planned) owns the speaker's relation to the addressee, which selects register,
|
||||
meaning packet owns what is meant, the scene state (live: `weksa scene put|dump`, labelled by the rule in
|
||||
`docs/scene-annotation.md`; speaker documents are planned) owns the speaker's relation to the addressee, which selects register,
|
||||
address and politeness, and a lowerer owns surface text and leaves a trace.
|
||||
|
||||
**The problem it solves.** An agent's outward speech today is one English
|
||||
|
|
|
|||
|
|
@ -1039,7 +1039,7 @@ fn committed_scenes_cover_original_groups() {
|
|||
let held: BTreeSet<String> = scenes.iter().map(|s| s.group.clone()).collect();
|
||||
assert_eq!(held, live, "a scene for each live Train or Validation group, none other");
|
||||
// Revisions append: a group's later revisions re-derived its relation.
|
||||
assert!(scenes.iter().all(|s| s.n <= 2));
|
||||
assert!(scenes.iter().all(|s| s.n <= 3));
|
||||
const S7: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated original scenes)";
|
||||
const GROWTH: &str = "claude-sonnet-5-5 (Weksa Hands, model-annotated validation-growth scenes; annotator is not the writer)";
|
||||
let grown: BTreeSet<String> = CorpusStore::open(&data.join("corpus.cc"))
|
||||
|
|
@ -1051,16 +1051,31 @@ fn committed_scenes_cover_original_groups() {
|
|||
.map(|i| i.group)
|
||||
.collect();
|
||||
assert_eq!(grown.len(), 42);
|
||||
const RUBRIC: &str =
|
||||
"claude-sonnet-5-5 (Weksa Hands, scene rubric r1, docs/scene-annotation.md)";
|
||||
assert!(
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../docs/scene-annotation.md")
|
||||
.is_file(),
|
||||
"the rubric r1 names a doc, and the doc is in the repo"
|
||||
);
|
||||
let mut by_growth = BTreeSet::new();
|
||||
for s in &scenes {
|
||||
match &s.annotator {
|
||||
Annotator::Model { family, generator } => {
|
||||
assert_eq!(*family, ModelFamily::Anthropic, "{}", s.group);
|
||||
assert!(generator == S7 || generator == GROWTH, "{}", s.group);
|
||||
assert!(
|
||||
generator == S7 || generator == GROWTH || generator == RUBRIC,
|
||||
"{}",
|
||||
s.group
|
||||
);
|
||||
if generator == GROWTH {
|
||||
by_growth.insert(s.group.clone());
|
||||
// One relation, keeper to unnamed, the three Scene features.
|
||||
// Written once, never revised by its own generator.
|
||||
assert_eq!(s.n, 1, "{}", s.group);
|
||||
}
|
||||
if generator == GROWTH || generator == RUBRIC {
|
||||
// One relation, keeper to unnamed, the three Scene features.
|
||||
assert_eq!(s.relations.len(), 1, "{}", s.group);
|
||||
let rel = &s.relations[0];
|
||||
assert_eq!(rel.from.as_str(), "entity:keeper", "{}", s.group);
|
||||
|
|
@ -1075,8 +1090,70 @@ fn committed_scenes_cover_original_groups() {
|
|||
by_growth, grown,
|
||||
"the growth scenes are exactly the vgrow groups"
|
||||
);
|
||||
// The label a run reads is each group's latest revision: all 98 are rubric r1's.
|
||||
let mut latest = std::collections::BTreeMap::new();
|
||||
for s in &scenes {
|
||||
if latest.get(&s.group).is_none_or(|l: &&weksa::speaker::Scene| l.n < s.n) {
|
||||
latest.insert(s.group.clone(), s);
|
||||
}
|
||||
}
|
||||
let mut lines = Vec::new();
|
||||
for (group, s) in &latest {
|
||||
let Annotator::Model { generator, .. } = &s.annotator else {
|
||||
panic!("{group} is not model-annotated");
|
||||
};
|
||||
assert_eq!(generator, RUBRIC, "{group}: the latest revision follows rubric r1");
|
||||
lines.push(format!("{group}\n{}\n{:?}", s.n, s.relations[0].features));
|
||||
}
|
||||
let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect();
|
||||
assert_eq!(
|
||||
weksa::split::fnv1a64_of(&parts),
|
||||
DIGEST_OF_THE_SCENES,
|
||||
"a latest scene changed without a rubric revision"
|
||||
);
|
||||
// The revisions before rubric r1 are provenance and never change: 113 of them.
|
||||
let mut earlier: Vec<&weksa::speaker::Scene> = scenes
|
||||
.iter()
|
||||
.filter(|s| !matches!(&s.annotator, Annotator::Model { generator, .. } if generator == RUBRIC))
|
||||
.collect();
|
||||
earlier.sort_by(|a, b| (&a.group, a.n).cmp(&(&b.group, b.n)));
|
||||
assert_eq!(earlier.len(), 113);
|
||||
let lines: Vec<String> = earlier
|
||||
.iter()
|
||||
.map(|s| format!("{}\n{}\n{:?}\n{:?}", s.group, s.n, s.annotator, s.relations))
|
||||
.collect();
|
||||
let parts: Vec<&[u8]> = lines.iter().map(|l| l.as_bytes()).collect();
|
||||
assert_eq!(
|
||||
weksa::split::fnv1a64_of(&parts),
|
||||
DIGEST_OF_THE_EARLIER_SCENES,
|
||||
"an earlier scene revision changed"
|
||||
);
|
||||
// Rubric r1 is text the labels cite: the section, as committed, is pinned.
|
||||
let doc = std::fs::read_to_string(
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../docs/scene-annotation.md"),
|
||||
)
|
||||
.unwrap();
|
||||
let from = doc.find("## Rubric r1").expect("the doc holds Rubric r1");
|
||||
let rest = &doc[from + "## Rubric r1".len()..];
|
||||
let section = &doc[from..from + "## Rubric r1".len() + rest.find("\n## ").expect("a section follows r1")];
|
||||
assert_eq!(
|
||||
weksa::split::fnv1a64_of(&[section.trim().as_bytes()]),
|
||||
DIGEST_OF_THE_RUBRIC,
|
||||
"rubric r1 was edited; a new rubric is a new section and a new generator"
|
||||
);
|
||||
}
|
||||
|
||||
/// Taken over the 113 scene revisions that precede rubric r1, 2026-11-13.
|
||||
const DIGEST_OF_THE_EARLIER_SCENES: u64 = 2598828882325200913;
|
||||
|
||||
/// Taken over the text of the `## Rubric r1` section of docs/scene-annotation.md
|
||||
/// (trimmed), 2026-11-13.
|
||||
const DIGEST_OF_THE_RUBRIC: u64 = 11823581212652524671;
|
||||
|
||||
/// Taken over each live group's latest revision (group, n, features) at rubric r1,
|
||||
/// 2026-11-13.
|
||||
const DIGEST_OF_THE_SCENES: u64 = 2505219080875817063;
|
||||
|
||||
/// The sweep both seed-pair tests share: over a grid of grades per feature (each
|
||||
/// also ABSENT) and the production `Condition::holds`, no listed pair of one
|
||||
/// language's entries co-holds; and each witness entry fires where it should and
|
||||
|
|
|
|||
Binary file not shown.
116
docs/scene-annotation.md
Normal file
116
docs/scene-annotation.md
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
# Scene annotation
|
||||
|
||||
How the keeper scenes of `data/original/speaker-state.cc` are labelled. A scene
|
||||
label has a committed rule that produced it; this file is that rule.
|
||||
|
||||
## Why
|
||||
|
||||
Two findings forced a written rule.
|
||||
|
||||
- `annotation-rank-from-register`: an annotator who reads the line's register
|
||||
(politeness forms, honorifics, endings) to decide the speaker's rank hands the
|
||||
lowering its own answer back, because register is what the lowering must
|
||||
produce from the scene.
|
||||
- The addressee gate (`with_scene` in `crates/weksa/src/validate.rs`): a scene is
|
||||
read only when the packet names a discourse addressee. A line that addresses no
|
||||
one reads `NoAddressee` and no Scene entry selects, so labels on narration are
|
||||
inert and the only labels that bite are those on lines said to someone.
|
||||
|
||||
The earlier labels followed two briefs that were never committed and that
|
||||
disagreed with each other (the relation to the reader, and a situation read with
|
||||
the register ignored). The rubric below replaces both for every live group.
|
||||
|
||||
## Provenance
|
||||
|
||||
Cut `weksa-lowering:cut_spec:cut-scene-in-group.r1`, ruling
|
||||
`weksa-lowering:ruling:scene-rubric-whole-bed`: all 98 live Train and Validation
|
||||
groups of `data/original` receive a new scene revision from annotator A under
|
||||
rubric r1, generator `claude-sonnet-5-5 (Weksa Hands, scene rubric r1,
|
||||
docs/scene-annotation.md)`. Earlier revisions stay as provenance. The rubric uses
|
||||
no corpus line as an example on purpose: a labelled example would make the blind
|
||||
check against the old labels circular.
|
||||
|
||||
A later rubric revision gets a new section and a new generator string; r1 is
|
||||
never edited.
|
||||
|
||||
## Rubric r1
|
||||
|
||||
The speaker of every line is the keeper, a quiet caretaker of a house and garden. First decide whether the line addresses a listener: it asks them something, asks them to do something, tells them about themselves, wishes something for them, or speaks to them as you. A line that addresses no one (narration, description, a remark about the world or about other people, a thought or wish with no one addressed) is told to the reader, a stranger: answer addressed false, relative_status level, intimacy 500, in_group false. People named inside a line are not its listener.
|
||||
|
||||
For a line that addresses a listener, answer addressed true and judge the keeper's relation to that listener from the situation the line states or implies, never from its register: politeness forms, honorifics, sentence endings, contractions, please and casual particles are what the lowering must produce from the scene, so they cannot decide it. relative_status: above when the situation gives the keeper standing over the listener (instructing, advising or permitting as elder, host, employer or teacher); below when it gives the listener standing over the keeper (the keeper serves, thanks, or asks a favour of a guest, employer, elder or official); level when it shows neither.
|
||||
|
||||
intimacy, 0 to 1000: 500 when the line shows nothing; up to 800 when it shows shared daily life (the listener's body, sleep or habits, the household's routine); above 800 only for a stated close bond; down to 200 for a stated first meeting or a service relation. in_group: true only when the line shows the listener belongs to the keeper's household, family or working circle, stated or plainly implied; a request, a question or a kind wish alone is not enough; otherwise false. Answer one JSON line per group, in the order given and nothing else: {"group": G, "addressed": true|false, "relative_status": S, "intimacy": N, "in_group": true|false}.
|
||||
|
||||
## Procedure
|
||||
|
||||
- Annotator A (committed) and annotator B (measurement only) are separate fresh
|
||||
model agents, one pair per language (49 live groups each). Each sees only the
|
||||
text of Rubric r1 above and that language's sentences as `group<TAB>"text"`,
|
||||
shuffled by a fixed seed. Neither sees an existing scene, the other annotator's
|
||||
answer, the ontology, which items are old or new, or any Test text.
|
||||
- An output that does not parse or misses a group is re-drawn once and logged.
|
||||
- A's labels are written with `weksa scene put` (one revision per group,
|
||||
`--to unnamed --inventory-rev 4`). `addressed` is not stored: no Scene feature
|
||||
carries it.
|
||||
- Disagreements are reported, never adjudicated.
|
||||
|
||||
## Agreement at r1 (this cut, 98 live groups, 49 per language)
|
||||
|
||||
Four fresh agents drew the labels (A-en, A-ja, B-en, B-ja); the briefs were the
|
||||
rubric section plus one language's shuffled sentences (SHA-256 of the brief
|
||||
bodies: en-US `db087476...dbc04e`, ja-JP `c5b81b27...97488c`; no Test text in
|
||||
either). No draw was repeated. Annotator model: `claude-sonnet-5-5`.
|
||||
|
||||
A against B (B measures only): pooled `addressed` equal 96/98 (kappa 0.96),
|
||||
`in_group` equal 93/98 (kappa 0.79), `relative_status` equal 89/98, intimacy
|
||||
within 150 on 98/98. en-US: addressed 47/49, in_group 47/49 (0.86),
|
||||
relative_status 45/49. ja-JP: addressed 49/49, in_group 46/49 (0.64),
|
||||
relative_status 44/49.
|
||||
|
||||
Blind check, A against the 56 older labels: `relative_status` equal 49/56,
|
||||
`in_group` equal 53/56 (kappa 0.64), intimacy within 150 on 55/56. Three
|
||||
in_group differences: `original-ja-shimoyado-2` and `original-ja-sakana` (both
|
||||
narrations, true to false) and `original-ja-asa` (an instruction to a household
|
||||
member, false to true).
|
||||
|
||||
In_group true across the 98: 42 before, 15 after; every unaddressed line is now
|
||||
false (it was true on 17 of the 21 unaddressed vgrow groups). Disagreements are
|
||||
reported, not adjudicated.
|
||||
|
||||
## Reproducing the cut
|
||||
|
||||
`scripts/scene-annotation/` holds the standard-library scripts of this cut, run
|
||||
from the repo root with `WEKSA` (the binary) and `SCENE_SCRATCH` (a directory
|
||||
that is never committed): `build_briefs.py` writes the two annotator briefs from
|
||||
the Rubric r1 section and prints each brief's Test-leak count (must be 0) and
|
||||
sha256 (it reproduces the briefs of this cut byte for byte); `put_labels.py`
|
||||
puts A's `A-en.jsonl` and `A-ja.jsonl`; `agreement.py` prints the tables above and
|
||||
must run before the puts, against the store at 76fa685, because the older labels
|
||||
are then its latest revisions (it also reads the round-1 encoder answers of
|
||||
`runs/growth-base` through `ROUND1`). `DIGEST_OF_THE_RUBRIC` and
|
||||
`DIGEST_OF_THE_EARLIER_SCENES` in `crates/weksa/tests/original_corpus.rs` pin the
|
||||
rubric text and the 113 earlier revisions.
|
||||
|
||||
## Known limits of r1 labels
|
||||
|
||||
Found by Soul's pass on this cut (`weksa-lowering:verdict:cut-scene-in-group.s1`).
|
||||
They are not fixed in r1; a reviewer's pass over the labels is follow-up
|
||||
`weksa-lowering:follow_up:scene-labels-reader-review`.
|
||||
|
||||
- The new labels change which ontology entries are selected for 37 groups. Eight
|
||||
addressed lines move onto the casual in-group entries (en-US:12: lantern,
|
||||
broom, mirror, seedling; ja-JP:12: furo, asa, irori, yurikago), while their
|
||||
references are polite requests. The reader default (level, 500, false) is
|
||||
exactly the condition of ja-JP:11.
|
||||
- `relative_status` tracks request form for some lines: imperatives read above,
|
||||
favour questions read level or below, so the same household request gets
|
||||
different standing in two forms (hane above/650/true, kabin below/500/false;
|
||||
B gave both level/false). Form is what signals the speech act, which is what
|
||||
part 2 asks for, and register is what part 2 forbids.
|
||||
- `in_group` is true on some requests alone (lantern, mulch, stool, hane) and on
|
||||
a line whose only family member is named inside it (original-ja-ane), though r1
|
||||
says a request or a person named inside the line is not enough. A and B
|
||||
agreeing does not clear this.
|
||||
- A run opened on the old speaker-state refuses to continue after merge:
|
||||
`runs/growth-base` pins the store of `/var/lib/mind-weksa-self/Weksa-growth`
|
||||
(main), and that checkout must not be pulled while the run is wanted.
|
||||
99
scripts/scene-annotation/agreement.py
Normal file
99
scripts/scene-annotation/agreement.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
"""Agreement of annotator A, B, the older labels and the round-1 encoder. Run it BEFORE the
|
||||
rubric r1 revisions are put: the older labels are the store's latest revision then.
|
||||
Standard library only. Usage: as build_briefs.py, plus ROUND1=<round-1 dir of the run>."""
|
||||
import re,sys,subprocess,json,glob,os
|
||||
import os
|
||||
W=os.environ.get("WEKSA","target/debug/weksa") # the weksa binary
|
||||
R=os.getcwd() # run from the repo root
|
||||
S=os.environ.get("SCENE_SCRATCH","scene-scratch")+"/" # A-*.jsonl, B-*.jsonl, briefs; never committed
|
||||
A=["--corpus","data/original/corpus.cc","--entities","data/original/entity.cc","--inventory","data/original/inventory.cc","--speaker-state","data/original/speaker-state.cc"]
|
||||
def run(*a): return subprocess.run([W]+A+list(a),cwd=R,capture_output=True,text=True).stdout.split("\n")
|
||||
out=run("corpus","dump")
|
||||
items=[]
|
||||
for i,l in enumerate(out):
|
||||
m=re.match(r'^item:(\S+) \[(\S+) \S+ group=(\S+) (\w+)\]$',l)
|
||||
if m and m.group(4)!="Test":
|
||||
t=out[i+1].strip(); t=json.loads(t) if '\\' in t else t[1:-1]
|
||||
items.append(dict(id=m.group(1),lang=m.group(2),group=m.group(3),text=t,new=m.group(1).startswith("vgrow")))
|
||||
G={x['group']:x for x in items}
|
||||
assert len(G)==98
|
||||
# scene dump: latest and @1 per group
|
||||
sd=run("scene","dump"); lab={}
|
||||
for i,l in enumerate(sd):
|
||||
m=re.match(r'^scene:(\S+)@(\d+) rev:\d+ Model \S+ (.*) \[entity:keeper\]$',l)
|
||||
if m:
|
||||
f=re.search(r'relative_status=(\w+) intimacy=(\d+) in_group=(\w+)',sd[i+1])
|
||||
lab.setdefault(m.group(1),{})[int(m.group(2))]=dict(rs=f.group(1),im=int(f.group(2)),ig=f.group(3)=="true",gen=m.group(3))
|
||||
old={g:v[max(v)] for g,v in lab.items()}
|
||||
def load(f): return {json.loads(l)['group']:json.loads(l) for l in open(S+f+".jsonl")}
|
||||
Aa={**load("A-en"),**load("A-ja")}; Bb={**load("B-en"),**load("B-ja")}
|
||||
assert set(Aa)==set(G)==set(Bb)
|
||||
def kappa(x,y):
|
||||
n=len(x);
|
||||
if n==0: return float('nan')
|
||||
po=sum(a==b for a,b in zip(x,y))/n
|
||||
pa=sum(x)/n; pb=sum(y)/n
|
||||
pe=pa*pb+(1-pa)*(1-pb)
|
||||
return float('nan') if pe==1 else (po-pe)/(1-pe)
|
||||
def tab(x,y):
|
||||
return "[TT=%d TF=%d FT=%d FF=%d]"%(sum(a and b for a,b in zip(x,y)),sum(a and not b for a,b in zip(x,y)),sum((not a) and b for a,b in zip(x,y)),sum((not a) and (not b) for a,b in zip(x,y)))
|
||||
def cmp(name,P,Q,gs,bkey=None):
|
||||
# P,Q: dict group-> record with keys addressed(optional) relative_status intimacy in_group
|
||||
def rs(r): return r.get('relative_status',r.get('rs'))
|
||||
def im(r): return r.get('intimacy',r.get('im'))
|
||||
def ig(r): return r.get('in_group',r.get('ig'))
|
||||
n=len(gs)
|
||||
if n==0: print(f"{name}: n=0"); return
|
||||
e=lambda f: sum(f(P[g])==f(Q[g]) for g in gs)
|
||||
s=f"{name}: n={n} rs_eq={e(rs)}/{n} ig_eq={e(ig)}/{n} im<=150={sum(abs(im(P[g])-im(Q[g]))<=150 for g in gs)}/{n}"
|
||||
s+=f" ig_kappa={kappa([ig(P[g]) for g in gs],[ig(Q[g]) for g in gs]):.2f} ig_tab(A.row,B.col)={tab([ig(P[g]) for g in gs],[ig(Q[g]) for g in gs])}"
|
||||
if all('addressed' in P[g] and 'addressed' in Q[g] for g in gs):
|
||||
s+=f" addr_eq={e(lambda r:r['addressed'])}/{n} addr_kappa={kappa([P[g]['addressed'] for g in gs],[Q[g]['addressed'] for g in gs]):.2f} addr_tab={tab([P[g]['addressed'] for g in gs],[Q[g]['addressed'] for g in gs])}"
|
||||
print(s)
|
||||
L={"en-US":[g for g in G if G[g]['lang']=="en-US"],"ja-JP":[g for g in G if G[g]['lang']=="ja-JP"],"pooled":list(G)}
|
||||
print("== (1) A vs B")
|
||||
for k,gs in L.items(): cmp(k,Aa,Bb,gs)
|
||||
oldg=[g for g in G if not G[g]['new']]; newg=[g for g in G if G[g]['new']]
|
||||
print("== (2) blind check: A vs old latest, the 56")
|
||||
for k,gs in L.items():
|
||||
gg=[g for g in gs if g in oldg]
|
||||
cmp(k+" all",Aa,old,gg)
|
||||
cmp(k+" A.addr=true",Aa,old,[g for g in gg if Aa[g]['addressed']])
|
||||
cmp(k+" A.addr=false",Aa,old,[g for g in gg if not Aa[g]['addressed']])
|
||||
print("differing in_group (old->A):")
|
||||
for g in oldg:
|
||||
if old[g]['ig']!=Aa[g]['in_group']: print(" ",g,old[g]['ig'],"->",Aa[g]['in_group'],"addr",Aa[g]['addressed'])
|
||||
print("differing any label, old vs A:")
|
||||
for g in oldg:
|
||||
o=old[g];a=Aa[g]
|
||||
if (o['rs'],o['im'],o['ig'])!=(a['relative_status'],a['intimacy'],a['in_group']): print(" %s old %s/%d/%s A %s/%d/%s addr=%s"%(g,o['rs'],o['im'],o['ig'],a['relative_status'],a['intimacy'],a['in_group'],a['addressed']))
|
||||
print("== (3) A vs vgrow @1, the 42")
|
||||
for k,gs in L.items(): cmp(k,Aa,{g:lab[g][1] for g in newg},[g for g in gs if g in newg])
|
||||
print("== (4) A addressed vs round-1 encoder addressee")
|
||||
enc={}
|
||||
RD=os.environ.get("ROUND1","runs/growth-base/round-1")+"/"
|
||||
for p in glob.glob(RD+"*.prompt"):
|
||||
pt=open(p).read()
|
||||
if not pt.startswith("Encode the line"): continue
|
||||
ap=p[:-7]+".answer"
|
||||
if not os.path.exists(ap): continue
|
||||
# the line text; take the item whose text appears in the prompt
|
||||
hit=[g for g in G if G[g]['text'] in pt]
|
||||
if len(hit)!=1: continue
|
||||
try: a=json.load(open(ap))
|
||||
except Exception: continue
|
||||
enc[hit[0]]=a.get('discourse',{}).get('addressee') is not None
|
||||
print("encoder answers matched:",len(enc),"of 98")
|
||||
for k,gs in list(L.items()):
|
||||
gg=[g for g in gs if g in enc]
|
||||
print(k,"n=",len(gg),"A.row,enc.col",tab([Aa[g]['addressed'] for g in gg],[enc[g] for g in gg]),"kappa=%.2f"%kappa([Aa[g]['addressed'] for g in gg],[enc[g] for g in gg]))
|
||||
for nm,grp in (("old",oldg),("new",newg)):
|
||||
gg=[g for g in grp if g in enc]; print(nm,"enc-addressee true:",sum(enc[g] for g in gg),"of",len(gg),"; A addressed true among old/new:",sum(Aa[g]['addressed'] for g in grp),"of",len(grp))
|
||||
print("== (5) in_group true counts, old/new x addressed, before -> after")
|
||||
for nm,grp in (("old56",oldg),("new42",newg)):
|
||||
for ad in (True,False):
|
||||
gg=[g for g in grp if Aa[g]['addressed']==ad]
|
||||
before=sum(old[g]['ig'] for g in gg); after=sum(Aa[g]['in_group'] for g in gg)
|
||||
print(f"{nm} A.addressed={ad}: n={len(gg)} in_group true before={before} after={after}")
|
||||
print("totals before",sum(old[g]['ig'] for g in G),"after",sum(Aa[g]['in_group'] for g in G))
|
||||
print("A addressed true:",sum(Aa[g]['addressed'] for g in G),"B:",sum(Bb[g]['addressed'] for g in G))
|
||||
37
scripts/scene-annotation/build_briefs.py
Normal file
37
scripts/scene-annotation/build_briefs.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""Build the two annotator briefs (rubric r1 section + one language's shuffled live
|
||||
sentences), print the Test-leak count of each (must be 0) and its sha256.
|
||||
Standard library only. Usage: from the repo root, WEKSA=... SCENE_SCRATCH=dir python3 -I scripts/scene-annotation/build_briefs.py"""
|
||||
import re,sys,subprocess,random,json,hashlib,os
|
||||
import os
|
||||
W=os.environ.get("WEKSA","target/debug/weksa") # the weksa binary
|
||||
R=os.getcwd() # run from the repo root
|
||||
S=os.environ.get("SCENE_SCRATCH","scene-scratch")+"/" # A-*.jsonl, B-*.jsonl, briefs; never committed
|
||||
A=["--corpus","data/original/corpus.cc","--entities","data/original/entity.cc","--inventory","data/original/inventory.cc","--speaker-state","data/original/speaker-state.cc"]
|
||||
out=subprocess.run([W]+A+["corpus","dump"],cwd=R,capture_output=True,text=True).stdout.split("\n")
|
||||
live=[];test=[]
|
||||
for i,l in enumerate(out):
|
||||
m=re.match(r'^item:(\S+) \[(\S+) \S+ group=(\S+) (\w+)\]$',l)
|
||||
if not m: continue
|
||||
t=out[i+1].strip()
|
||||
assert t.startswith('"') and t.endswith('"')
|
||||
t=json.loads(t) if '\\' in t else t[1:-1]
|
||||
if m.group(4)=="Test": test.append(t)
|
||||
else: live.append((m.group(2),m.group(3),t))
|
||||
assert len(live)==98,len(live)
|
||||
rub=open(R+"/docs/scene-annotation.md").read()
|
||||
sec=rub[rub.index("## Rubric r1"):]
|
||||
# rubric section ends at next "## " heading
|
||||
rest=sec[len("## Rubric r1"):]
|
||||
nx=rest.find("\n## ")
|
||||
sec=sec[:len("## Rubric r1")+(nx if nx>=0 else len(rest))].strip()+"\n"
|
||||
def leaks(brief,text):
|
||||
w=text.split(); c=list(text)
|
||||
return text in brief or any(" ".join(w[k:k+5]) in brief for k in range(len(w)-4)) or (len(w)==1 and any("".join(c[k:k+12]) in brief for k in range(len(c)-11)))
|
||||
for lang in ("en-US","ja-JP"):
|
||||
rows=[(g,t) for (l,g,t) in live if l==lang]
|
||||
assert len(rows)==49
|
||||
random.Random(f"scene-r1-{lang}").shuffle(rows)
|
||||
body=sec+"\nThe groups, one per line as group<TAB>\"text\":\n\n"+"\n".join(f'{g}\t"{t}"' for g,t in rows)+"\n"
|
||||
n=sum(leaks(body,t) for t in test)
|
||||
print(lang,"lines",len(rows),"test_leaks",n,hashlib.sha256(body.encode()).hexdigest())
|
||||
open(S+f"brief-{lang}.txt","w").write(body)
|
||||
15
scripts/scene-annotation/put_labels.py
Normal file
15
scripts/scene-annotation/put_labels.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"""Put annotator A's labels through weksa scene put, one revision per group.
|
||||
Standard library only. Usage: as build_briefs.py."""
|
||||
import json,subprocess
|
||||
import os
|
||||
W=os.environ.get("WEKSA","target/debug/weksa") # the weksa binary
|
||||
R=os.getcwd() # run from the repo root
|
||||
S=os.environ.get("SCENE_SCRATCH","scene-scratch")+"/" # A-*.jsonl, B-*.jsonl, briefs; never committed
|
||||
A=["--corpus","data/original/corpus.cc","--entities","data/original/entity.cc","--inventory","data/original/inventory.cc","--speaker-state","data/original/speaker-state.cc"]
|
||||
rows=[json.loads(l) for f in ("A-en","A-ja") for l in open(S+f"{f}.jsonl")]
|
||||
for r in sorted(rows,key=lambda r:r['group']):
|
||||
c=[W]+A+["scene","put","--group",r['group'],"--to","unnamed","--inventory-rev","4",
|
||||
"--set",f"relative_status={r['relative_status']}","--set",f"intimacy={r['intimacy']}","--set",f"in_group={'true' if r['in_group'] else 'false'}",
|
||||
"--model-family","anthropic","--generator","claude-sonnet-5-5 (Weksa Hands, scene rubric r1, docs/scene-annotation.md)"]
|
||||
p=subprocess.run(c,cwd=R,capture_output=True,text=True)
|
||||
print(r['group'],p.returncode,p.stdout.strip()[:80],p.stderr.strip()[:200])
|
||||
Loading…
Reference in a new issue