crates/weksa/src/main.rs

@@ -15,7 +15,7 @@ use weksa::packet::{FeatValue, FeatureValue, Grade};
 use weksa::packet::proof_packet;
 use weksa::model::{ModelPort, RecordedAnswers};
 use weksa::run::{
-    AblationOutcome, CycleOutcome, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
+    AblationOutcome, CalibrateOutcome, CycleOutcome, FLOOR_ARMS_MIN, LowererReach, OntologyMintOutcome, RecallOutcome, RelowerOutcome, RoundOutcome, RunPaths, RunStores, cycle, round,
 };
 use weksa::score::ScoreInput;
 use weksa::{
@@ -174,6 +174,31 @@ enum LoopCommand {
         #[arg(long, default_value = "http://127.0.0.1:11434")]
         ollama_url: String,
     },
+    /// Replay the accept rule over answered relower draws of round ROUND, to
+    /// measure how often it accepts, rejects or regresses what it should. Reads
+    /// the recorded lowering and each draw (--prompted: repeats of it, at least
+    /// four; --degraded: arms that should regress; --candidate: arms to rate),
+    /// answered through `weksa loop relower ROOT --round N --arm LABEL`, and
+    /// prints counts over all Validation items and over each --class
+    /// NAME=PREFIX[,PREFIX...] (items whose key starts with a prefix). Writes
+    /// no prompts; exits 1 naming the draws still unanswered. Procedure:
+    /// docs/calibration.md. A diagnostic, never a gate; records nothing; Test
+    /// and Train are never touched. Labels match [a-z0-9-]{1,32} and differ.
+    Calibrate {
+        root: PathBuf,
+        #[arg(long)]
+        round: u32,
+        #[arg(long)]
+        prompted: Vec<String>,
+        #[arg(long)]
+        degraded: Vec<String>,
+        #[arg(long)]
+        candidate: Vec<String>,
+        #[arg(long)]
+        class: Vec<String>,
+        #[arg(long, default_value = "http://127.0.0.1:11434")]
+        ollama_url: String,
+    },
     /// Run one edit-accept cycle of the run at ROOT. The stores are read only
     /// from the run's record. The editor's prompt, then the candidate round's
     /// encode prompts, then its lowering prompts are written in turn; the
@@ -660,6 +685,12 @@ impl AddArgs {
     }
 }
 
+/// A draw or arm label: [a-z0-9-]{1,32}.
+fn is_label(l: &str) -> bool {
+    (1..=32).contains(&l.len())
+        && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
+}
+
 /// The split of an item as `Train`, `Validation`, `Test` or `excluded`.
 fn split_text(item: &CorpusItem) -> String {
     SplitScheme::V1
@@ -1146,12 +1177,8 @@ fn main() -> Result<()> {
             floor,
             ollama_url,
         }) => {
-            let label = |l: &str| {
-                (1..=32).contains(&l.len())
-                    && l.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
-            };
-            if !label(&arm)
-                || floor.iter().any(|f| !label(f) || *f == arm)
+            if !is_label(&arm)
+                || floor.iter().any(|f| !is_label(f) || *f == arm)
                 || floor.iter().enumerate().any(|(i, f)| floor[..i].contains(f))
             {
                 anyhow::bail!("arm and floor labels must match [a-z0-9-]{{1,32}} and differ");
@@ -1197,6 +1224,95 @@ fn main() -> Result<()> {
                 }
             }
         }
+        Command::Loop(LoopCommand::Calibrate {
+            root,
+            round,
+            prompted,
+            degraded,
+            candidate,
+            class,
+            ollama_url,
+        }) => {
+            let labels: Vec<&String> = prompted.iter().chain(&degraded).chain(&candidate).collect();
+            if labels.iter().any(|l| !is_label(l))
+                || labels.iter().enumerate().any(|(i, l)| labels[..i].contains(l))
+            {
+                anyhow::bail!("draw labels must match [a-z0-9-]{{1,32}} and differ");
+            }
+            if prompted.len() < FLOOR_ARMS_MIN + 1 {
+                anyhow::bail!("at least {} --prompted labels are needed", FLOOR_ARMS_MIN + 1);
+            }
+            let mut classes: Vec<(String, Vec<String>)> = Vec::new();
+            for c in &class {
+                let Some((name, prefixes)) = c.split_once('=') else {
+                    anyhow::bail!("--class is NAME=PREFIX[,PREFIX...]");
+                };
+                if !is_label(name)
+                    || name == "all"
+                    || classes.iter().any(|(n, _)| n == name)
+                    || prefixes.split(',').any(str::is_empty)
+                {
+                    anyhow::bail!("--class is NAME=PREFIX[,PREFIX...] with a new label NAME other than all");
+                }
+                classes.push((name.to_string(), prefixes.split(',').map(String::from).collect()));
+            }
+            let run = RunStores::open(&root)?;
+            let scorer = Scorer::new(OllamaEmbedder::new(&ollama_url, &run.record().embedder));
+            match run.calibrate(round, &prompted, &degraded, &candidate, &classes, &scorer)? {
+                CalibrateOutcome::Unanswered(missing) => {
+                    for (label, n) in &missing {
+                        println!("unanswered {label} {n}");
+                    }
+                    eprintln!(
+                        "answer it through weksa loop relower ROOT --round N --arm LABEL first"
+                    );
+                    std::process::exit(1);
+                }
+                CalibrateOutcome::Read { unusable, classes } => {
+                    println!(
+                        "{}",
+                        run.labelled(&format!(
+                            "calibrate draws prompted={} degraded={} candidate={} floor-arms={FLOOR_ARMS_MIN} unusable={unusable}",
+                            prompted.len(),
+                            degraded.len(),
+                            candidate.len(),
+                        ))
+                    );
+                    for c in classes {
+                        let items: String = c
+                            .languages
+                            .iter()
+                            .map(|l| format!(" {} n={} skipped={}", l.language, l.n, l.skipped))
+                            .collect();
+                        let t = &c.trials;
+                        println!("{}", run.labelled(&format!("calibrate class={} items{items}", c.name)));
+                        println!(
+                            "{}",
+                            run.labelled(&format!(
+                                "calibrate class={} prompted trials={} nogain={} accepted={} regressed={} noise={:.4}",
+                                c.name, t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed, t.noise
+                            ))
+                        );
+                        for (label, d) in degraded.iter().zip(&t.degraded) {
+                            println!(
+                                "{}",
+                                run.labelled(&format!(
+                                    "calibrate class={} degraded={label} trials={} regressed={}",
+                                    c.name, d.trials, d.regressed
+                                ))
+                            );
+                        }
+                        println!(
+                            "{}",
+                            run.labelled(&format!(
+                                "calibrate class={} candidate trials={} accepted={} nogain={} regressed={}",
+                                c.name, t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed
+                            ))
+                        );
+                    }
+                }
+            }
+        }
         Command::Loop(LoopCommand::Recall { root }) => {
             let run = RunStores::open(&root)?;
             let mut files = run.answer_files(run.recall_dir())?;

crates/weksa/src/run.rs

@@ -44,7 +44,7 @@ use crate::encode::{
     EditDraft, LoweringDraft, MAX_EDIT_CHANGES, OntologyDraft, PacketDraft, encoder_id,
 };
 use crate::form::{is_attestation, is_language_tag, is_slug};
-use crate::model::{AnswerFiles, ModelPort, PortError};
+use crate::model::{AnswerFiles, ModelPort, PortError, RecordedAnswers};
 use crate::packet::{Packet, Stance};
 use crate::score::{CHRF_VERSION, chrf, Embedder, ItemScore, ScoreInput, Scorer, SplitMean, aggregate, validation_items};
 use crate::split::{Split, SplitScheme, fnv1a64, fnv1a64_of, hex16};
@@ -1675,27 +1675,90 @@ impl RunStores {
         floors: &mut [&mut dyn ModelPort],
         scorer: &Scorer<E>,
     ) -> Result<RelowerOutcome, RunError> {
-        let store = |_| RunError::Store;
         check_embedder(self, scorer)?;
-        let held = self.held_splits(n, &[Split::Validation])?;
-        let inventory = self.inventory.inventory().map_err(store)?;
-        let ontology = self.ontology.ontology().map_err(store)?;
-
-        let dir = self.relower_dir(n, arm_label);
-        std::fs::create_dir_all(&dir).map_err(|e| RunError::Io(e.kind()))?;
-        for suffix in ["cc", "cc.lock"] {
-            let _ = std::fs::remove_file(dir.join(format!("base.{suffix}")));
-        }
-        let mut scratch = PacketStore::open(&dir.join("base.cc")).map_err(store)?;
-
         if (1..FLOOR_ARMS_MIN).contains(&floors.len()) {
             return Err(RunError::FloorArms);
         }
-        // Per language, per item: the item text, then the recorded and arm
-        // lowerings and one per floor arm (`None` when unusable).
-        type Row = (String, Option<String>, Option<String>, Vec<Option<String>>);
-        let mut waiting: Vec<String> = Vec::new();
-        let mut floor_waiting = false;
+        let dir = self.relower_dir(n, arm_label);
+        // Ports in the order the prompts are asked: recorded, floors, arm.
+        let floor_count = floors.len();
+        let mut ports: Vec<&mut dyn ModelPort> = vec![recorded];
+        for f in floors.iter_mut() {
+            let f: &mut dyn ModelPort = &mut **f;
+            ports.push(f);
+        }
+        ports.push(arm);
+        let drawn = self.read_draws(n, &dir.join("base.cc"), &mut ports, scorer)?;
+        let arm_at = floor_count + 1;
+        if drawn.waiting[1..arm_at].iter().any(|w| !w.is_empty()) {
+            return Err(RunError::FloorUnanswered);
+        }
+        if !drawn.waiting[arm_at].is_empty() {
+            return Ok(RelowerOutcome::Unanswered { prompts: drawn.waiting[arm_at].clone() });
+        }
+        let mut languages = Vec::new();
+        for l in &drawn.languages {
+            let of_port = |p: usize| -> Vec<ItemScore> { l.items.iter().map(|(_, s)| s[p].clone()).collect() };
+            let rec = of_port(0);
+            let floor_scores: Vec<Vec<ItemScore>> = (1..arm_at).map(of_port).collect();
+            languages.push(RelowerReading {
+                language: l.language.clone(),
+                n: l.items.len() as u32,
+                skipped: l.skipped,
+                unusable: l.unusable[arm_at],
+                floor_unusable: l.unusable[1..arm_at].iter().sum(),
+                recorded: reading_of(&rec)?,
+                arm: reading_of(&of_port(arm_at))?,
+                floor: if floor_count == 0 { None } else { Some(paired_floor(&rec, &floor_scores)) },
+            });
+        }
+        let verdict = if floor_count > 0 {
+            let panel: Vec<(Reading, Reading, Reading)> = languages
+                .iter()
+                .filter_map(|l| Some((l.recorded, l.arm, l.floor?)))
+                .collect();
+            Some(decide_panel(&panel))
+        } else {
+            None
+        };
+        Ok(RelowerOutcome::Read { languages, verdict })
+    }
+
+    /// Asks every port, in the order given, each live Validation item's
+    /// byte-identical lowering prompt of round `n` and scores the answers against
+    /// the items in one [`Scorer::score`] call: the one owner of per-item
+    /// scoring for [`relower`](Self::relower) and
+    /// [`calibrate`](Self::calibrate). An answer that is not a usable lowering
+    /// scores 0. Port 0 is the recorded lowering, which must be answered
+    /// (`Unrecorded` at once); any other port that is not is listed in
+    /// `waiting`, and nothing is scored. `scratch` is the packet store the
+    /// prompts are built in, removed and reopened here; its directory is
+    /// created here, after the round is found.
+    fn read_draws<E: Embedder>(
+        &self,
+        n: u32,
+        scratch: &Path,
+        ports: &mut [&mut dyn ModelPort],
+        scorer: &Scorer<E>,
+    ) -> Result<Drawn, RunError> {
+        let store = |_| RunError::Store;
+        let held = self.held_splits(n, &[Split::Validation])?;
+        let inventory = self.inventory.inventory().map_err(store)?;
+        let ontology = self.ontology.ontology().map_err(store)?;
+        if let Some(dir) = scratch.parent() {
+            std::fs::create_dir_all(dir).map_err(|e| RunError::Io(e.kind()))?;
+        }
+        for suffix in ["", ".lock"] {
+            let mut name = scratch.as_os_str().to_owned();
+            name.push(suffix);
+            let _ = std::fs::remove_file(PathBuf::from(name));
+        }
+        let mut scratch = PacketStore::open(scratch).map_err(store)?;
+
+        // Per language, per item: the item id and text, then each port's lowering
+        // (`None` when unusable or unanswered).
+        let mut waiting: Vec<Vec<String>> = vec![Vec::new(); ports.len()];
+        type Row = (String, String, Vec<Option<String>>);
         let mut rows: Vec<Vec<Row>> = Vec::new();
         for (_, _, items) in &held {
             let mut language_rows = Vec::new();
@@ -1707,49 +1770,33 @@ impl RunStores {
                 else {
                     continue;
                 };
-                let text = |a: Asked| match a {
-                    Asked::Text(t) => Some(t),
-                    _ => None,
-                };
-                let recorded_text = match asked(recorded, &prompt)? {
-                    Asked::Waiting => return Err(RunError::Unrecorded),
-                    other => text(other),
-                };
-                let mut floor_texts = Vec::new();
-                for f in floors.iter_mut() {
-                    floor_texts.push(match asked(&mut **f, &prompt)? {
+                let mut texts = Vec::new();
+                for (p, port) in ports.iter_mut().enumerate() {
+                    texts.push(match asked(&mut **port, &prompt)? {
+                        Asked::Text(t) => Some(t),
+                        Asked::Unusable => None,
+                        Asked::Waiting if p == 0 => return Err(RunError::Unrecorded),
                         Asked::Waiting => {
-                            floor_waiting = true;
+                            let d = prompt.digest();
+                            if !waiting[p].contains(&d) {
+                                waiting[p].push(d);
+                            }
                             None
                         }
-                        other => text(other),
                     });
                 }
-                let arm_text = match asked(arm, &prompt)? {
-                    Asked::Waiting => {
-                        let d = prompt.digest();
-                        if !waiting.contains(&d) {
-                            waiting.push(d);
-                        }
-                        None
-                    }
-                    other => text(other),
-                };
-                language_rows.push((item.text.clone(), recorded_text, arm_text, floor_texts));
+                language_rows.push((item.id.as_str().to_string(), item.text.clone(), texts));
             }
             rows.push(language_rows);
         }
-        if floor_waiting {
-            return Err(RunError::FloorUnanswered);
-        }
-        if !waiting.is_empty() {
-            return Ok(RelowerOutcome::Unanswered { prompts: waiting });
+        if waiting.iter().any(|w| !w.is_empty()) {
+            return Ok(Drawn { languages: Vec::new(), waiting });
         }
 
         // One score call: each usable lowering against its item, in row order.
         let mut inputs: Vec<ScoreInput> = Vec::new();
-        for (reference, recorded, arm, floor_texts) in rows.iter().flatten() {
-            for output in [recorded, arm].into_iter().chain(floor_texts).flatten() {
+        for (_, reference, texts) in rows.iter().flatten() {
+            for output in texts.iter().flatten() {
                 inputs.push(ScoreInput { reference, output });
             }
         }
@@ -1759,67 +1806,272 @@ impl RunStores {
             scorer.score(&inputs).map_err(|_| RunError::Embed)?
         }
         .into_iter();
-        let mut next = |t: &Option<String>| -> Result<ItemScore, RunError> {
-            match t {
-                Some(_) => scores.next().ok_or(RunError::Store),
-                None => Ok(ItemScore { chrf: 0.0, meaning: Some(0.0) }),
-            }
-        };
-        let reading_of = |scored: &[ItemScore]| {
-            Reading::of(&SplitMean {
-                count: scored.len(),
-                chrf: mean(scored.iter().map(|s| s.chrf)),
-                meaning: mean(scored.iter().filter_map(|s| s.meaning)),
-            })
-            .ok_or(RunError::NoValidation)
-        };
-        let mut languages = Vec::new();
         let live = self.corpus.all().map_err(store)?;
-        for ((language, _, _), language_rows) in held.iter().zip(&rows) {
-            let (mut rec, mut arm_scores) = (Vec::new(), Vec::new());
-            let mut floor_scores: Vec<Vec<ItemScore>> = vec![Vec::new(); floors.len()];
-            let (mut unusable, mut floor_unusable) = (0u32, 0u32);
-            for (_, r, a, f) in language_rows {
-                rec.push(next(r)?);
-                arm_scores.push(next(a)?);
-                if a.is_none() {
-                    unusable += 1;
-                }
-                for (scores, t) in floor_scores.iter_mut().zip(f) {
-                    scores.push(next(t)?);
-                    if t.is_none() {
-                        floor_unusable += 1;
-                    }
+        let mut languages = Vec::new();
+        for ((language, _, _), language_rows) in held.iter().zip(rows) {
+            let mut unusable = vec![0u32; ports.len()];
+            let mut items = Vec::new();
+            for (id, _, texts) in language_rows {
+                let mut item_scores = Vec::new();
+                for (p, t) in texts.iter().enumerate() {
+                    item_scores.push(match t {
+                        Some(_) => scores.next().ok_or(RunError::Store)?,
+                        None => {
+                            unusable[p] += 1;
+                            ItemScore { chrf: 0.0, meaning: Some(0.0) }
+                        }
+                    });
                 }
+                items.push((id, item_scores));
             }
-            let recorded_reading = reading_of(&rec)?;
-            languages.push(RelowerReading {
+            let live_validation = live
+                .iter()
+                .filter(|i| &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation))
+                .count() as u32;
+            languages.push(DrawnLanguage {
                 language: language.clone(),
-                n: language_rows.len() as u32,
-                skipped: live
-                    .iter()
-                    .filter(|i| {
-                        &i.language == language && SplitScheme::V1.live(i) == Some(Split::Validation)
-                    })
-                    .count() as u32
-                    - language_rows.len() as u32,
+                skipped: live_validation - items.len() as u32,
+                items,
                 unusable,
-                floor_unusable,
-                recorded: recorded_reading,
-                arm: reading_of(&arm_scores)?,
-                floor: if floors.is_empty() { None } else { Some(paired_floor(&rec, &floor_scores)) },
             });
         }
-        let verdict = if !floors.is_empty() {
-            let panel: Vec<(Reading, Reading, Reading)> = languages
-                .iter()
-                .filter_map(|l| Some((l.recorded, l.arm, l.floor?)))
-                .collect();
-            Some(decide_panel(&panel))
-        } else {
-            None
+        Ok(Drawn { languages, waiting })
+    }
+}
+
+/// One panel language of [`RunStores::read_draws`].
+struct DrawnLanguage {
+    language: String,
+    /// Live Validation items of the language with no lowering prompt.
+    skipped: u32,
+    /// Per read item in key order: its id and one score per port.
+    items: Vec<(String, Vec<ItemScore>)>,
+    /// Per port, answers that were not a usable lowering.
+    unusable: Vec<u32>,
+}
+
+struct Drawn {
+    /// Per panel language in panel order; empty while any port is waiting.
+    languages: Vec<DrawnLanguage>,
+    /// Per port, the digests of the prompts it holds no answer to.
+    waiting: Vec<Vec<String>>,
+}
+
+/// A reading over per-item scores; `NoValidation` without items.
+fn reading_of(scored: &[ItemScore]) -> Result<Reading, RunError> {
+    Reading::of(&SplitMean {
+        count: scored.len(),
+        chrf: mean(scored.iter().map(|s| s.chrf)),
+        meaning: mean(scored.iter().filter_map(|s| s.meaning)),
+    })
+    .ok_or(RunError::NoValidation)
+}
+
+/// How trials of one kind read: counts, not rates.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Counts {
+    pub trials: u32,
+    pub accepted: u32,
+    pub nogain: u32,
+    pub regressed: u32,
+}
+
+impl Counts {
+    fn tally(&mut self, v: Verdict) {
+        self.trials += 1;
+        match v {
+            Verdict::Accepted => self.accepted += 1,
+            Verdict::Rejected(Why::Regressed(_)) => self.regressed += 1,
+            Verdict::Rejected(_) => self.nogain += 1,
+        }
+    }
+}
+
+/// What [`calibration_trials`] counted over one set of items.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Trials {
+    /// Ordered (base, candidate) pairs of prompted draws, each over every floor.
+    pub prompted: Counts,
+    /// Per degraded arm in the order given: the arm as the candidate.
+    pub degraded: Vec<Counts>,
+    /// Every candidate arm as the candidate, pooled.
+    pub candidate: Counts,
+    /// The mean of [`panel_noise`] over every (base, floor) choice.
+    pub noise: f64,
+}
+
+/// Every `k`-subset of `pool`, in order.
+fn subsets(pool: &[usize], k: usize) -> Vec<Vec<usize>> {
+    if k == 0 {
+        return vec![Vec::new()];
+    }
+    let mut out = Vec::new();
+    for (i, &first) in pool.iter().enumerate() {
+        for mut rest in subsets(&pool[i + 1..], k - 1) {
+            rest.insert(0, first);
+            out.push(rest);
+        }
+    }
+    out
+}
+
+/// The accept rule replayed over draws that are all lowerings of one round, as
+/// the live loop would meet them: [`decide_panel`] over (base, candidate,
+/// [`paired_floor`] of the base against `FLOOR_ARMS_MIN` other prompted draws),
+/// one row per language. `draws` is `[port][language][item]`; ports
+/// `0..prompted` are prompted draws (the recorded lowering first), then
+/// `degraded` and `candidate` ports follow. A language with no items reads
+/// NaN, which fails closed.
+pub fn calibration_trials(
+    draws: &[Vec<Vec<ItemScore>>],
+    prompted: usize,
+    degraded: usize,
+    candidate: usize,
+) -> Trials {
+    let languages = draws.first().map_or(0, Vec::len);
+    let rows = |b: usize, c: usize, floors: &[usize]| -> Vec<(Reading, Reading, Reading)> {
+        let read = |scores: &[ItemScore]| {
+            reading_of(scores).unwrap_or(Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN })
         };
-        Ok(RelowerOutcome::Read { languages, verdict })
+        (0..languages)
+            .map(|l| {
+                let arms: Vec<Vec<ItemScore>> = floors.iter().map(|&f| draws[f][l].clone()).collect();
+                (read(&draws[b][l]), read(&draws[c][l]), paired_floor(&draws[b][l], &arms))
+            })
+            .collect()
+    };
+    let pool: Vec<usize> = (0..prompted).collect();
+    let without = |out: &[usize]| -> Vec<usize> { pool.iter().copied().filter(|p| !out.contains(p)).collect() };
+    let (mut prompted_counts, mut candidate_counts) = (Counts::default(), Counts::default());
+    let mut degraded_counts = vec![Counts::default(); degraded];
+    let (mut noise, mut noises) = (0.0, 0u32);
+    for &b in &pool {
+        for floors in subsets(&without(&[b]), FLOOR_ARMS_MIN) {
+            for (d, counts) in degraded_counts.iter_mut().enumerate() {
+                counts.tally(decide_panel(&rows(b, prompted + d, &floors)));
+            }
+            for c in prompted + degraded..prompted + degraded + candidate {
+                candidate_counts.tally(decide_panel(&rows(b, c, &floors)));
+            }
+            noise += panel_noise(&rows(b, b, &floors));
+            noises += 1;
+        }
+        for &c in pool.iter().filter(|&&c| c != b) {
+            for floors in subsets(&without(&[b, c]), FLOOR_ARMS_MIN) {
+                prompted_counts.tally(decide_panel(&rows(b, c, &floors)));
+            }
+        }
+    }
+    Trials {
+        prompted: prompted_counts,
+        degraded: degraded_counts,
+        candidate: candidate_counts,
+        noise: noise / f64::from(noises),
+    }
+}
+
+/// One panel language of a class: items read and live Validation items of the
+/// class not read.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClassLanguage {
+    pub language: String,
+    pub n: u32,
+    pub skipped: u32,
+}
+
+/// The trials over one class of items.
+#[derive(Clone, Debug, PartialEq)]
+pub struct ClassTrials {
+    pub name: String,
+    pub languages: Vec<ClassLanguage>,
+    pub trials: Trials,
+}
+
+#[derive(Debug)]
+pub enum CalibrateOutcome {
+    /// Draw labels with the number of prompts each has no answer to.
+    Unanswered(Vec<(String, u32)>),
+    /// `unusable` over every prompted, degraded and candidate draw; `all` first,
+    /// then the classes in the order given.
+    Read { unusable: u32, classes: Vec<ClassTrials> },
+}
+
+impl RunStores {
+    /// Replays the accept rule over answered relower draws: reads every draw of
+    /// round `n` (the recorded lowering and the `relower-<n>-<label>`
+    /// directories, through [`RecordedAnswers`] only, so nothing is written
+    /// there) with [`read_draws`](Self::read_draws), and counts
+    /// [`calibration_trials`] over all items and over each class, a class being
+    /// the items whose key starts with any of its prefixes. See
+    /// `docs/calibration.md`. Records nothing; the scratch packet store is
+    /// `<root>/calibrate-<n>/base.cc`.
+    pub fn calibrate<E: Embedder>(
+        &self,
+        n: u32,
+        prompted: &[String],
+        degraded: &[String],
+        candidate: &[String],
+        classes: &[(String, Vec<String>)],
+        scorer: &Scorer<E>,
+    ) -> Result<CalibrateOutcome, RunError> {
+        check_embedder(self, scorer)?;
+        let labels: Vec<&String> = prompted.iter().chain(degraded).chain(candidate).collect();
+        let mut recorded =
+            RecordedAnswers::new(self.root.join(format!("round-{n}")), self.record.model.clone());
+        let mut draws: Vec<RecordedAnswers> = labels
+            .iter()
+            .map(|l| RecordedAnswers::new(self.relower_dir(n, l), self.record.model.clone()))
+            .collect();
+        let mut ports: Vec<&mut dyn ModelPort> = vec![&mut recorded];
+        for d in draws.iter_mut() {
+            ports.push(d);
+        }
+        let dir = self.root.join(format!("calibrate-{n}"));
+        let drawn = self.read_draws(n, &dir.join("base.cc"), &mut ports, scorer)?;
+        let missing: Vec<(String, u32)> = labels
+            .iter()
+            .zip(&drawn.waiting[1..])
+            .filter(|(_, w)| !w.is_empty())
+            .map(|(l, w)| ((*l).clone(), w.len() as u32))
+            .collect();
+        if !missing.is_empty() {
+            return Ok(CalibrateOutcome::Unanswered(missing));
+        }
+        let live = self.corpus.all().map_err(|_| RunError::Store)?;
+        let everything = [("all".to_string(), vec![String::new()])];
+        let mut out = Vec::new();
+        for (name, prefixes) in everything.iter().chain(classes) {
+            let keep = |id: &str| prefixes.iter().any(|p| id.starts_with(p.as_str()));
+            let mut by_port: Vec<Vec<Vec<ItemScore>>> = vec![Vec::new(); ports.len()];
+            let mut languages = Vec::new();
+            for l in &drawn.languages {
+                let items: Vec<&Vec<ItemScore>> =
+                    l.items.iter().filter(|(id, _)| keep(id)).map(|(_, s)| s).collect();
+                for (p, port) in by_port.iter_mut().enumerate() {
+                    port.push(items.iter().map(|s| s[p].clone()).collect());
+                }
+                let in_class = live
+                    .iter()
+                    .filter(|i| {
+                        i.language == l.language
+                            && SplitScheme::V1.live(i) == Some(Split::Validation)
+                            && keep(i.id.as_str())
+                    })
+                    .count() as u32;
+                languages.push(ClassLanguage {
+                    language: l.language.clone(),
+                    n: items.len() as u32,
+                    skipped: in_class - items.len() as u32,
+                });
+            }
+            out.push(ClassTrials {
+                name: name.clone(),
+                languages,
+                trials: calibration_trials(&by_port, prompted.len() + 1, degraded.len(), candidate.len()),
+            });
+        }
+        let unusable = drawn.languages.iter().map(|l| l.unusable[1..].iter().sum::<u32>()).sum();
+        Ok(CalibrateOutcome::Read { unusable, classes: out })
     }
 }
 
@@ -2258,8 +2510,11 @@ fn nano(x: f64) -> i64 {
 /// otherwise read as unchanged); a metric that falls by more than its
 /// tolerance (the floor, at least one step) regresses that metric; the
 /// composite falling by more than max(its floor, one [`LADDER_STEP`])
-/// regresses the composite. Means count a refused Validation item as 0, so
-/// refusing a weak item cannot raise a reading. Differences are compared in
+/// regresses the composite. Regression compares the readings it is given
+/// and counts nothing itself: a round's reading counts a refused item as 0
+/// ([`round`], `score_round`), so refusing a weak item cannot
+/// raise it, while a relower or calibrate reading is over the items that have a
+/// lowering prompt only and counts the rest in `skipped`, not as 0. Differences are compared in
 /// billionths ([`nano`]), so a fall of exactly the tolerance is a tie.
 fn regression(base: &Reading, candidate: &Reading, floor: &Reading) -> Option<Metric> {
     let finite = |r: &Reading| r.chrf.is_finite() && r.meaning.is_finite() && r.composite.is_finite();
@@ -2327,16 +2582,21 @@ pub fn decide_panel(rows: &[(Reading, Reading, Reading)]) -> Verdict {
         .iter()
         .map(|(base, candidate, _)| nano(candidate.composite - base.composite))
         .sum();
-    // The noise of a sum of independent languages' differences: the root of the
-    // sum of their floors squared.
-    let noise = rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::<f64>().sqrt();
-    if net > nano(noise.max(LADDER_STEP)) {
+    if net > nano(panel_noise(rows).max(LADDER_STEP)) {
         Verdict::Accepted
     } else {
         Verdict::Rejected(Why::NoGain)
     }
 }
 
+/// The noise of a sum of independent languages' differences: the root of the
+/// sum of their composite floors squared, over one `(base, candidate, floor)`
+/// row per language. A one-language panel's noise is its floor. The one owner
+/// of the term, for [`decide_panel`] and the calibration.
+pub fn panel_noise(rows: &[(Reading, Reading, Reading)]) -> f64 {
+    rows.iter().map(|(_, _, floor)| floor.composite.powi(2)).sum::<f64>().sqrt()
+}
+
 /// One panel language's row of a cycle: the head's Validation reading, the
 /// candidate's (none when the edit never reached a round), the language's
 /// measured floor and the threshold derived from it. The threshold decides only
@@ -2570,7 +2830,7 @@ pub const FLOOR_ARMS_MIN: usize = 3;
 /// than two items or no arm (0/0), and when an arm is ragged.
 pub fn paired_floor(recorded: &[ItemScore], arms: &[Vec<ItemScore>]) -> Reading {
     let n = recorded.len();
-    if arms.iter().any(|a| a.len() != n) {
+    if n < 2 || arms.iter().any(|a| a.len() != n) {
         return Reading { chrf: f64::NAN, meaning: f64::NAN, composite: f64::NAN };
     }
     let pooled = |of: &dyn Fn(&ItemScore) -> f64| {
@@ -2780,6 +3040,52 @@ fn discard_inventory(dir: &Path) {
 mod tests {
     use super::*;
 
+    fn scored(chrf: f64, meaning: f64) -> ItemScore {
+        ItemScore { chrf, meaning: Some(meaning) }
+    }
+
+    /// A draw over `[language][item]`: `base` plus a per-draw, per-item wobble.
+    fn draw(languages: usize, items: usize, base: f64, wobble: f64, seed: usize) -> Vec<Vec<ItemScore>> {
+        (0..languages)
+            .map(|l| {
+                (0..items)
+                    .map(|i| {
+                        let w = wobble * (((i * 7 + seed * 3 + l) % 5) as f64 - 2.0) / 2.0;
+                        scored((100.0 * (base + w)).clamp(0.0, 100.0), (base + w).clamp(0.0, 1.0))
+                    })
+                    .collect()
+            })
+            .collect()
+    }
+
+    #[test]
+    fn calibration_trials_count_every_choice_and_read_the_fixed_cases() {
+        // Six prompted draws, two degraded, three candidates, two languages of 8 items.
+        let mut draws: Vec<_> = (0..6).map(|s| draw(2, 8, 0.6, 0.02, s)).collect();
+        draws.extend((0..2).map(|_| draw(2, 8, 0.0, 0.0, 0)));
+        draws.extend((0..3).map(|s| draw(2, 8, 0.8, 0.02, s)));
+        let t = calibration_trials(&draws, 6, 2, 3);
+        assert_eq!(t.prompted.trials, 120);
+        assert_eq!(t.degraded.iter().map(|d| d.trials).collect::<Vec<_>>(), [60, 60]);
+        assert_eq!(t.candidate.trials, 6 * 10 * 3);
+        assert!(t.noise > 0.0);
+        // An all-zero draw regresses against any base, in every trial.
+        assert!(t.degraded.iter().all(|d| d.regressed == d.trials));
+        // A far better draw is accepted in every trial.
+        assert_eq!(t.candidate.accepted, t.candidate.trials);
+
+        // Identical prompted draws: nothing to gain, nothing regressed, no noise.
+        let same: Vec<_> = (0..6).map(|_| draw(2, 8, 0.6, 0.02, 1)).collect();
+        let t = calibration_trials(&same, 6, 0, 0);
+        assert_eq!((t.prompted.trials, t.prompted.nogain), (120, 120));
+        assert_eq!(t.noise, 0.0);
+
+        // A class of one item has no floor: every trial fails closed.
+        let one: Vec<_> = (0..6).map(|s| draw(2, 1, 0.6, 0.02, s)).collect();
+        let t = calibration_trials(&one, 6, 0, 0);
+        assert_eq!((t.prompted.trials, t.prompted.regressed), (120, 120));
+    }
+
     /// The twin of the `RoundRecord` literal in its compile_fail doctest: the
     /// same fields, compiling here because the fields are private to this module.
     #[test]

crates/weksa/tests/cli.rs

@@ -1866,6 +1866,99 @@ fn cli_loop_relower_exits_3_until_answered_and_refuses_bad_labels() {
     }
 }
 
+#[test]
+fn cli_loop_calibrate_reads_answered_draws_and_refuses_unanswered_and_bad_labels() {
+    let d = tempfile::tempdir().unwrap();
+    open_dao_run(d.path());
+    let url = fake_embedder();
+    let root = d.path().join("canaryroot");
+    let mut run = weksa::run::RunStores::open(&root).unwrap();
+    let scorer = weksa::Scorer::new(weksa::OllamaEmbedder::new(&url, &run.record().embedder));
+    let weksa::run::RoundOutcome::Recorded(r) = weksa::run::round(&mut run, &mut DaoModel, &scorer).unwrap()
+    else {
+        panic!("answered")
+    };
+    let said = r#"{"text":"The highest excellence is like the water."}"#;
+    for call in &r.calls()[3..] {
+        std::fs::write(root.join(format!("round-1/{}.answer", call.prompt)), said).unwrap();
+    }
+    let calibrate = |extra: &[&str]| {
+        weksa(
+            d.path(),
+            &[&["loop", "calibrate", "canaryroot", "--round", "1", "--ollama-url", &url], extra].concat(),
+        )
+    };
+    let four = ["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4"];
+
+    // Unanswered draws: exit 1, each label with its missing count, no relower directory created.
+    let o = calibrate(&four);
+    assert_eq!(o.status.code(), Some(1), "{}", out(&o));
+    let text = out(&o);
+    for label in ["p1", "p2", "p3", "p4"] {
+        assert!(text.contains(&format!("unanswered {label} 2")), "{text}");
+        assert!(!root.join(format!("relower-1-{label}")).exists());
+    }
+    assert!(text.contains("answer it through weksa loop relower"), "{text}");
+
+    // Answer every draw through relower's door; from here the only thing that can refuse is the argument.
+    let o = weksa(
+        d.path(),
+        &["loop", "relower", "canaryroot", "--round", "1", "--lowerer", "x", "--arm", "p1", "--ollama-url", &url],
+    );
+    assert_eq!(o.status.code(), Some(3), "{}", out(&o));
+    let digests: Vec<String> = out(&o)
+        .lines()
+        .filter_map(|l| l.strip_prefix("unanswered "))
+        .filter(|x| x.len() == 16)
+        .map(String::from)
+        .collect();
+    assert_eq!(digests.len(), 2);
+    // `P1` and `a_b` are answered too, so only the label check can refuse them.
+    for label in ["p1", "p2", "p3", "p4", "bad", "P1", "a_b"] {
+        std::fs::create_dir_all(root.join(format!("relower-1-{label}"))).unwrap();
+        for digest in &digests {
+            std::fs::write(root.join(format!("relower-1-{label}/{digest}.answer")), said).unwrap();
+        }
+    }
+    // A draw answered in part.
+    std::fs::create_dir_all(root.join("relower-1-part")).unwrap();
+    std::fs::write(root.join(format!("relower-1-part/{}.answer", digests[0])), said).unwrap();
+    let o = calibrate(&["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "part"]);
+    assert_eq!(o.status.code(), Some(1), "{}", out(&o));
+    assert!(out(&o).contains("unanswered part 1") && !out(&o).contains("unanswered p1"), "{}", out(&o));
+
+    // Refusals, every draw answered: the baseline reads, each variant differs by one fault.
+    assert!(calibrate(&four).status.success());
+    for args in [
+        vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3"],
+        vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p3"],
+        vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4", "--degraded", "p4"],
+        vec!["--prompted", "p1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4", "--candidate", "bad", "--degraded", "bad"],
+        vec!["--prompted", "P1", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4"],
+        vec!["--prompted", "a_b", "--prompted", "p2", "--prompted", "p3", "--prompted", "p4"],
+        [&four[..], &["--class", "all=item:"]].concat(),
+        [&four[..], &["--class", "noequals"]].concat(),
+        [&four[..], &["--class", "Old=item:a"]].concat(),
+        [&four[..], &["--class", "old=item:a,"]].concat(),
+        [&four[..], &["--class", "old=item:a", "--class", "old=item:b"]].concat(),
+    ] {
+        let o = calibrate(&args);
+        assert_eq!(o.status.code(), Some(1), "{args:?}: {}", out(&o));
+        assert!(!out(&o).contains("unanswered"), "{args:?} must be refused for itself: {}", out(&o));
+    }
+
+    let o = calibrate(&[&four[..], &["--degraded", "bad", "--class", "old=item:daodejing"]].concat());
+    assert!(o.status.success(), "{}", out(&o));
+    let text = out(&o);
+    assert!(text.starts_with("calibrate draws prompted=4 degraded=1 candidate=0 floor-arms=3 unusable=0"), "{text}");
+    assert!(text.contains("calibrate class=all items en-US n=2 skipped=0"), "{text}");
+    assert!(text.contains("calibrate class=all prompted trials=20 nogain=20 accepted=0 regressed=0"), "{text}");
+    assert!(text.contains("calibrate class=old degraded=bad trials=20 regressed=0"), "{text}");
+    for leaked in ["item:daodejing", "packet:", "highest excellence", "canaryroot"] {
+        assert!(!text.contains(leaked), "{text}");
+    }
+}
+
 #[test]
 fn loop_ablate_without_the_rounds_answers_says_so() {
     let d = tempfile::tempdir().unwrap();

crates/weksa/tests/golf_loop.rs

@@ -6174,3 +6174,257 @@ fn relower_arm_prompt_holding_test_text_is_withheld() {
         .unwrap_or(0);
     assert_eq!(prompts, 0, "no prompt file is written");
 }
+
+// ---- calibrate: the accept rule replayed over answered relower draws ----
+
+/// The cut a [`Tee`] answers with something that is no lowering.
+const NOT_A_LOWERING: usize = 99;
+
+/// Answers each lowering prompt with the item's own text cut by `cut(key)`
+/// tenths, and leaves the answer where an answer file goes, as the relay does.
+struct Tee {
+    dir: PathBuf,
+    texts: HashMap<String, String>,
+    cut: Box<dyn Fn(&str) -> usize>,
+}
+
+impl ModelPort for Tee {
+    fn model(&self) -> &str {
+        "fake-model"
+    }
+    fn answer(&mut self, prompt: &Prompt) -> Result<String, PortError> {
+        let key = prompt.item_keys()[0].clone();
+        let text: Vec<char> = self.texts[&key].chars().collect();
+        let cut = (self.cut)(&key);
+        let keep = (text.len() * (10 - cut.min(10))).div_ceil(10);
+        let answer = if cut == NOT_A_LOWERING {
+            "this is no lowering".to_string()
+        } else {
+            serde_json::json!({ "text": text[..keep].iter().collect::<String>() }).to_string()
+        };
+        std::fs::create_dir_all(&self.dir).unwrap();
+        std::fs::write(self.dir.join(format!("{}.answer", prompt.digest())), &answer).unwrap();
+        Ok(answer)
+    }
+}
+
+/// A bilingual Dao run grown to seven live Validation items per language, five
+/// of them `item:vgrow:*`, with round 1 recorded.
+fn grown_bilingual() -> Dao {
+    let mut dao = dao_panel(
+        |corpus, entities| {
+            add_languages(corpus, entities, LAOZI_JA);
+            let entities = weksa::EntityStore::open(entities).unwrap();
+            let mut store = CorpusStore::open(corpus).unwrap();
+            for k in 0..5 {
+                for (from, key, text, language) in [
+                    ("item:daodejing:2", format!("item:vgrow:{}", k + 1), format!("The sage grows item {k} by letting the thing alone for {} seasons", k * 3 + 1), "en-US"),
+                    ("item:daodejing:12", format!("item:vgrowja:{}", k + 1), format!("聖人は{}の季節を数えて物ごとに静かに任せる", "春夏秋冬".chars().cycle().skip(k).take(k + 2).collect::<String>()), "ja-JP"),
+                ] {
+                    let mut item = store.get(from).unwrap().unwrap();
+                    item.id = key.as_str().into();
+                    item.text = text;
+                    item.language = language.into();
+                    item.names = vec![];
+                    item.prior_turns = vec![];
+                    store.add(item, &entities).unwrap();
+                }
+            }
+        },
+        &["ja-JP", "en-US"],
+    );
+    let mut model = bilingual_fake();
+    for k in 0..5 {
+        for key in [format!("item:vgrow:{}", k + 1), format!("item:vgrowja:{}", k + 1)] {
+            model.answers.insert(key.clone(), citing_laozi(2));
+            model.lowered.insert(key, "placeholder".into());
+        }
+    }
+    recorded(round(&mut dao.run, &mut model, &Scorer::new(Bag)).unwrap());
+    dao
+}
+
+fn corpus_texts(dao: &Dao) -> HashMap<String, String> {
+    CorpusStore::open(&dao.corpus)
+        .unwrap()
+        .all()
+        .unwrap()
+        .into_iter()
+        .map(|i| (i.id.as_str().to_string(), i.text))
+        .collect()
+}
+
+/// A draw label's answers, written through relower's own door.
+fn answer_draw(dao: &Dao, label: &str, cut: impl Fn(&str) -> usize + 'static) {
+    use weksa::model::RecordedAnswers;
+    let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
+    let mut tee = Tee { dir: dao.run.relower_dir(1, label), texts: corpus_texts(dao), cut: Box::new(cut) };
+    dao.run.relower(1, label, &mut recorded, &mut tee, &mut [], &Scorer::new(Bag)).unwrap();
+}
+
+/// A stable per-item wobble in 0..4 for draw `d`.
+fn wobble(d: usize, key: &str) -> usize {
+    (key.bytes().map(usize::from).sum::<usize>() * (d + 3) + d) % 4
+}
+
+#[test]
+fn calibrate_reads_answered_draws_by_decide_panel() {
+    use weksa::run::CalibrateOutcome;
+    let dao = grown_bilingual();
+    // The recorded lowering's answers, through a tee over the round directory.
+    let mut round1 = Tee { dir: dao.root.join("round-1"), texts: corpus_texts(&dao), cut: Box::new(|k| wobble(0, k)) };
+    let mut seed = Tee { dir: dao.run.relower_dir(1, "seed"), texts: corpus_texts(&dao), cut: Box::new(|_| 0) };
+    dao.run.relower(1, "seed", &mut round1, &mut seed, &mut [], &Scorer::new(Bag)).unwrap();
+
+    let prompted: Vec<String> = (1..=5).map(|d| format!("p{d}")).collect();
+    for (i, label) in prompted.iter().enumerate() {
+        answer_draw(&dao, label, move |k| wobble(i + 1, k));
+    }
+    answer_draw(&dao, "bad", |_| 9);
+    answer_draw(&dao, "mild", |k| 1 + wobble(7, k) % 2);
+    for (i, label) in ["c1", "c2"].iter().enumerate() {
+        answer_draw(&dao, label, move |k| wobble(i + 10, k) / 2);
+    }
+    let degraded = ["bad".to_string(), "mild".to_string()];
+    let candidate = ["c1".to_string(), "c2".to_string()];
+    let classes = [
+        ("old".to_string(), vec!["item:daodejing".to_string()]),
+        ("new".to_string(), vec!["item:vgrow".to_string()]),
+    ];
+    let before = tree(&dao.root);
+    let CalibrateOutcome::Read { unusable, classes } = dao
+        .run
+        .calibrate(1, &prompted, &degraded, &candidate, &classes, &Scorer::new(Bag))
+        .unwrap()
+    else {
+        panic!("every draw is answered")
+    };
+    let after = tree(&dao.root);
+    assert!(
+        after.keys().all(|p| before.contains_key(p) || p.starts_with("calibrate-1/")),
+        "calibrate writes only its scratch store"
+    );
+    assert_eq!(unusable, 0);
+    let got: Vec<String> = classes
+        .iter()
+        .map(|c| {
+            let t = &c.trials;
+            let items: Vec<(u32, u32)> = c.languages.iter().map(|l| (l.n, l.skipped)).collect();
+            format!(
+                "{} {items:?} prompted {}/{}/{}/{} noise {:.4} degraded {:?} candidate {}/{}/{}/{}",
+                c.name,
+                t.prompted.trials, t.prompted.nogain, t.prompted.accepted, t.prompted.regressed,
+                t.noise,
+                t.degraded.iter().map(|d| (d.trials, d.regressed)).collect::<Vec<_>>(),
+                t.candidate.trials, t.candidate.accepted, t.candidate.nogain, t.candidate.regressed,
+            )
+        })
+        .collect();
+    assert_eq!(
+        got,
+        [
+            "all [(7, 0), (7, 0)] prompted 120/30/24/66 noise 0.0468 degraded [(60, 60), (60, 43)] candidate 120/64/40/16",
+            "old [(2, 0), (2, 0)] prompted 120/118/0/2 noise 0.1354 degraded [(60, 60), (60, 60)] candidate 120/0/120/0",
+            "new [(5, 0), (5, 0)] prompted 120/16/24/80 noise 0.0138 degraded [(60, 60), (60, 50)] candidate 120/60/20/40",
+        ]
+    );
+}
+
+/// An item in each language whose packet, so whose prompt, no other item shares.
+const BAD_EN: &str = "item:daodejing:3";
+const BAD_JA: &str = "item:daodejing:13";
+
+#[test]
+fn calibrate_classes_match_key_prefixes_exactly() {
+    use weksa::run::CalibrateOutcome;
+    let dao = grown_bilingual();
+    let seed = |dao: &Dao| {
+        let mut round1 = Tee { dir: dao.root.join("round-1"), texts: corpus_texts(dao), cut: Box::new(|_| 0) };
+        let mut arm = Tee { dir: dao.run.relower_dir(1, "seed"), texts: corpus_texts(dao), cut: Box::new(|_| 0) };
+        dao.run.relower(1, "seed", &mut round1, &mut arm, &mut [], &Scorer::new(Bag)).unwrap();
+    };
+    seed(&dao);
+    let prompted: Vec<String> = (1..=4).map(|d| format!("p{d}")).collect();
+    for label in &prompted {
+        answer_draw(&dao, label, |_| 0);
+    }
+    let class = |name: &str, prefix: &str| (name.to_string(), vec![prefix.to_string()]);
+    let classes = [
+        class("en", "item:vgrow:"),      // the en-US items only; a prefix one byte short also takes the ja-JP ones
+        class("mid", "vgrow"),           // a key does not start with it, though it contains it
+        class("both", "item:vgrow"),
+        ("two".to_string(), vec!["item:vgrow:1".to_string(), "item:vgrowja:1".to_string()]),
+    ];
+    let CalibrateOutcome::Read { classes, .. } =
+        dao.run.calibrate(1, &prompted, &[], &[], &classes, &Scorer::new(Bag)).unwrap()
+    else {
+        panic!("answered")
+    };
+    let seen: Vec<(String, Vec<(u32, u32)>)> = classes
+        .iter()
+        .map(|c| (c.name.clone(), c.languages.iter().map(|l| (l.n, l.skipped)).collect()))
+        .collect();
+    // Panel order: en-US, ja-JP.
+    assert_eq!(
+        seen,
+        [
+            ("all".to_string(), vec![(7, 0), (7, 0)]),
+            ("en".to_string(), vec![(5, 0), (0, 0)]),
+            ("mid".to_string(), vec![(0, 0), (0, 0)]),
+            ("both".to_string(), vec![(5, 0), (5, 0)]),
+            ("two".to_string(), vec![(1, 0), (1, 0)]),
+        ]
+    );
+}
+
+#[test]
+fn unusable_answers_are_counted_for_draws_and_for_arm_and_floors_apart() {
+    use weksa::model::RecordedAnswers;
+    use weksa::run::{CalibrateOutcome, RelowerOutcome};
+    let dao = grown_bilingual();
+    let tee = |dir: PathBuf, bad: &'static [&'static str]| Tee {
+        dir,
+        texts: corpus_texts(&dao),
+        cut: Box::new(move |k| if bad.contains(&k) { NOT_A_LOWERING } else { 0 }),
+    };
+    // The recorded lowering is unusable on one item; it is no draw and is not counted.
+    let mut round1 = tee(dao.root.join("round-1"), &[BAD_EN]);
+    let mut seed = tee(dao.run.relower_dir(1, "seed"), &[]);
+    dao.run.relower(1, "seed", &mut round1, &mut seed, &mut [], &Scorer::new(Bag)).unwrap();
+
+    // relower: the arm is unusable on one item, the first floor on two; they are counted apart.
+    let mut recorded = RecordedAnswers::new(dao.root.join("round-1"), "fake-model".into());
+    let mut arm = tee(dao.run.relower_dir(1, "arm"), &[BAD_EN]);
+    let mut f1 = tee(dao.run.relower_dir(1, "f1"), &[BAD_EN, BAD_JA]);
+    let mut f2 = tee(dao.run.relower_dir(1, "f2"), &[]);
+    let mut f3 = tee(dao.run.relower_dir(1, "f3"), &[]);
+    let RelowerOutcome::Read { languages, .. } = dao
+        .run
+        .relower(1, "arm", &mut recorded, &mut arm, &mut [&mut f1, &mut f2, &mut f3], &Scorer::new(Bag))
+        .unwrap()
+    else {
+        panic!("answered")
+    };
+    assert_eq!(languages.iter().map(|l| l.unusable).sum::<u32>(), 1);
+    assert_eq!(languages.iter().map(|l| l.floor_unusable).sum::<u32>(), 2);
+
+    // calibrate: three unusable answers across the draws, none of them the recorded one.
+    answer_draw_bad(&dao, "p1", &[BAD_EN, BAD_JA]);
+    answer_draw_bad(&dao, "p2", &[]);
+    answer_draw_bad(&dao, "p3", &[]);
+    answer_draw_bad(&dao, "p4", &[]);
+    answer_draw_bad(&dao, "bad", &[BAD_EN]);
+    let prompted: Vec<String> = (1..=4).map(|d| format!("p{d}")).collect();
+    let CalibrateOutcome::Read { unusable, .. } = dao
+        .run
+        .calibrate(1, &prompted, &["bad".to_string()], &[], &[], &Scorer::new(Bag))
+        .unwrap()
+    else {
+        panic!("answered")
+    };
+    assert_eq!(unusable, 3);
+}
+
+fn answer_draw_bad(dao: &Dao, label: &str, bad: &'static [&'static str]) {
+    answer_draw(dao, label, move |k| if bad.contains(&k) { NOT_A_LOWERING } else { 0 });
+}

docs/calibration.md

@@ -0,0 +1,99 @@
+# Calibrating the accept rule
+
+`weksa loop calibrate` measures how the live accept rule behaves on a run's own
+Validation corpus: how often it accepts, rejects as no gain, or regresses a
+candidate that should be none of those. It does not reimplement the rule. It
+reads the answered draws through `RunStores::read_draws` (the scoring
+`weksa loop relower` uses), decides every trial with `decide_panel` over
+`paired_floor` (`crates/weksa/src/run.rs`), and prints, per class, the items
+read and skipped per language, the trial totals with their Accepted, NoGain and
+Regressed counts, and the mean panel noise. That is all it produces: the
+Accepted/NoGain/Regressed counts and the noise mean in the map's "Recalibration
+on the grown corpus" block are reproduced by it; the other figures there (the
+35% with R removed, the noise range, the 0.0185 gain and its 0.41 ratio, the
+bootstrap intervals, the per-item SDs, R's +0.024, the mild-arm duplicate
+counts and mean composite offsets, the Train-vocabulary overlap, the old/new
+class composites and the ten parse-refused item ids) were computed by scratch scripts that are not in the repo. It is a
+diagnostic: it records nothing, writes no prompts, and never touches Test or
+Train.
+
+## What it counts
+
+The draws are lowerings of one round's Validation packets: the round's own
+(recorded) lowering `R`, plus draws you label. `P` is `R` and the `--prompted`
+draws; the floor size is `FLOOR_ARMS_MIN` (3), the live rule's.
+
+- Prompted trials: every ordered pair (base, candidate) in `P`, against every
+  3-subset of the rest of `P` as the floor. A repeat of the same lowering
+  should read NoGain; Accepted is a false accept, Regressed a false reject.
+  (`R` comes from the round, not from `relower`, so it may not be
+  exchangeable with a repeat; read the pairs that involve it with that in mind.)
+- Degraded trials, per `--degraded` arm: every base in `P` against every
+  3-subset of the rest of `P`, the degraded draw as the candidate. These should
+  read Regressed.
+- Candidate trials, pooled over the `--candidate` arms: the same, tallied
+  Accepted, NoGain, Regressed. This is the rule's power against that arm.
+- Noise: the mean `panel_noise` over every (base, floor) choice.
+
+With six prompted draws including `R` that is 120 prompted trials, 60 per
+degraded arm and 240 candidate trials. Trials share draws and are not
+independent. Each count is printed for `all` Validation items and for each
+`--class NAME=PREFIX[,PREFIX...]` (items whose key starts with any prefix), from
+the same per-item scores. A class with fewer than two read items in a language
+has no floor and reads Regressed in every trial; that is printed.
+
+## Steps
+
+1. `weksa loop open ROOT ...` and `weksa loop round ROOT` record round N and
+   leave its answers in `ROOT/round-N/`.
+2. For each draw label: `weksa loop relower ROOT --round N --arm LABEL
+   --lowerer NAME` writes the byte-identical lowering prompts to
+   `ROOT/relower-N-LABEL/<digest>.prompt` and exits 3. Answer each prompt into
+   `ROOT/relower-N-LABEL/<digest>.answer`.
+3. `weksa loop calibrate ROOT --round N --prompted p1 --prompted p2 ...
+   [--degraded LABEL]... [--candidate LABEL]... [--class NAME=PREFIX]...`.
+   At least four `--prompted` labels are needed (a pair and three floors with
+   `R` make five draws). It exits 1 and lists `unanswered LABEL COUNT` for any
+   draw with prompts still unanswered. Labels match `[a-z0-9-]{1,32}` and differ
+   across the three kinds. It needs the Ollama embedder of the run; a full
+   60-item, 13-draw calibration is on the order of 1500 embedded texts, so
+   launch it detached with a log.
+
+The scratch packet store is `ROOT/calibrate-N/base.cc`; delete it at will.
+
+## Answer file format
+
+`<digest>.answer` is one JSON object and nothing else, no code fence:
+`{"text": "..."}`, a JSON-escaped string. The answerer reads only the prompt
+(its first line says what to write) and writes the answer. Every prompt of the
+draw must be answered.
+
+## Briefs per arm
+
+Independence rule: a fresh agent per batch of answers; no batch sees another
+draw's answers. Only `mild` reads `round-N/*.answer`.
+
+- Plain prompted: answer each prompt as its own first line asks; read nothing
+  else in the run root.
+- Mild: for each prompt, read the recorded lowering in
+  `ROOT/round-N/<digest>.answer` and write the same-format answer whose text is
+  that utterance with exactly one meaning detail blurred: change or drop one of
+  tense, polarity, modality or hedge, a referent, a quantity or a predicate's
+  concept (replace it with a near but different word). Keep language, register,
+  length and fluency; the result must read as a natural utterance and must not
+  mention the change. Pick the detail that is not the most salient one. A
+  second mild draw blurs a different detail than the first and is written
+  without reading the first's answers. The wording of the distil-base original
+  is lost, and the dispatch used for growth-base is not on disk, so this is the
+  wording from now on. Warning: on growth-base two mild arms briefed this way
+  came out identical in 27 of 60 items, so independently briefed mild arms
+  are not independent degradations; read a mild draw's regress count with that
+  in mind.
+- Train-guide: before answering, read the Train sentences of the corpus in the
+  batch's language as a register guide and let it steer word choice; the guide
+  gives phrasing only and each answer must still say exactly what its own packet
+  says. The guide section is the live Train items of that language, built from
+  `weksa corpus dump` filtered to Train, and is given to the answerer in the
+  brief, never pasted into this document, and never includes Validation or Test
+  text. Read nothing else in the run root.
+- Control: a script writes `{"text":"Yes."}` as the answer to every prompt.

docs/targets/interlingua-lowering-map.md

@@ -505,11 +505,24 @@ findings shaped the wording.
   `validation-growth-writer-claude-checked`). The new scenes use a different
   `in_group` convention from the old ones (follow-up
   `vgrow-scene-in-group-convention`), so the new-item scores are for scenes that
-  will be re-annotated; the corpus digest changes again then.
+  will be re-annotated; the speaker-state store changes then (a run pins its
+  digest), the corpus does not.
 
-Evidence lives in the Self workspace (`cal2/`, `runs/growth-base`), not in git,
-and uses a DUMP patch kept out of the repo; no committed harness reruns these
-numbers (Soul finding `evidence-outside-git`, follow-up owed).
+`weksa loop calibrate` (`docs/calibration.md`) reproduces from `run.rs`'s own
+scoring and accept rule the counts per class, the trial totals and the mean
+noise of this block: prompted 116 NoGain, 4 Accepted, 0 Regressed of 120;
+control-yes 60 of 60 Regressed, mild-1 19 of 60, mild-2 0 of 60; the
+Train-guide arm 51 Accepted, 189 NoGain, 0 Regressed of 240; noise 0.0451;
+items en-US n=28 skipped=7, ja-JP n=32 skipped=3. By class it adds: old items
+(`item:original*`) prompted 111/9/0, noise 0.1069, Train-guide 71 Accepted of
+240; new items (`item:vgrow*`) prompted 116/4/0, noise 0.0334, Train-guide 0
+Accepted and 74 Regressed of 240. The other figures of this block (35% with R
+removed, the noise range, the 0.0185 gain and 0.41 ratio, the bootstrap
+intervals, the per-item SDs, R's +0.024, the mild-arm duplicate counts and
+mean composite offsets, the Train-vocabulary overlap, the old/new class
+composites, the ten parse-refused item ids) were
+computed by scratch scripts outside git (Soul finding `evidence-outside-git`)
+and are not reproduced by the harness.
 
 ### Smallest measurable distil slice (`distil-incontext`, not built)