From c47e8e415d6b2f86824f6b9377b00359502eaf8a Mon Sep 17 00:00:00 2001 From: Eureka Date: Thu, 29 Oct 2026 14:34:00 +0100 Subject: [PATCH] packet-store-corpus (2b): truthful corpus labels with ModelFamily and verbatim generator, whole-item S1, canonical keys, S6 no copy under a new key (squash of weksa-lowering/packet-store-corpus 491536c..f9258d5; Soul s1) Co-Authored-By: Claude Opus 5.5 --- crates/weksa/src/corpus.rs | 33 ++- crates/weksa/src/main.rs | 38 +++- crates/weksa/src/store.rs | 35 +++- crates/weksa/tests/cli.rs | 39 +++- crates/weksa/tests/common/mod.rs | 4 +- crates/weksa/tests/split_schema.rs | 5 +- crates/weksa/tests/stores.rs | 198 +++++++++++++++++- data/corpus/corpus.cc | 2 +- .../brokkr-edda-voice/lowered-output.yaml | 2 +- schemas/cultnet/weksa.corpus_item.schema.json | 35 +++- 10 files changed, 354 insertions(+), 37 deletions(-) diff --git a/crates/weksa/src/corpus.rs b/crates/weksa/src/corpus.rs index ee7e447..2b04c11 100644 --- a/crates/weksa/src/corpus.rs +++ b/crates/weksa/src/corpus.rs @@ -29,12 +29,28 @@ pub enum Source { edition_url: String, }, /// Labelled model output; never in the test split, never judged by its own family. + /// `family` is `Unrecorded` when nothing records one. `generator` is the + /// string the fixture or run wrote, kept verbatim. ModelWritten { - model_family: String, + family: ModelFamily, + generator: Option, prompt_digest: Option, }, } +/// The family of the model that wrote a line. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum ModelFamily { + Anthropic, + OpenAi, + Google, + Alibaba, + Meta, + Mistral, + /// Nothing records which family wrote the line. + Unrecorded, +} + /// Key: `item::`. #[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)] #[cultcache(type = "weksa.corpus_item")] @@ -68,10 +84,19 @@ pub fn item_key(slug: &str, n: u32) -> String { format!("item:{slug}:{n}") } -/// The slug of an item key, if it is `item::`. +/// The slug of an item key, if it is canonical `item::`: a slug of +/// `[a-z0-9]` and inner `-` (at most 48 bytes, no edge `-`), and `n` a decimal +/// of at least 1 without sign or leading zero that fits `u32`. pub fn key_slug(key: &str) -> Option<&str> { let rest = key.strip_prefix("item:")?; let (slug, n) = rest.rsplit_once(':')?; - n.parse::().ok()?; - (!slug.is_empty()).then_some(slug) + let canonical_n = + n.bytes().all(|b| b.is_ascii_digit()) && !n.starts_with('0') && n.parse::().is_ok(); + let b = slug.as_bytes(); + let slug_ok = (1..=48).contains(&b.len()) + && b.iter() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') + && b[0] != b'-' + && b[b.len() - 1] != b'-'; + (canonical_n && slug_ok).then_some(slug) } diff --git a/crates/weksa/src/main.rs b/crates/weksa/src/main.rs index 7887488..e86c2f1 100644 --- a/crates/weksa/src/main.rs +++ b/crates/weksa/src/main.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use anyhow::{Context, Result, bail}; use clap::{Args, Parser, Subcommand, ValueEnum}; -use weksa::corpus::{CorpusItem, Source, Turn}; +use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn}; use weksa::inventory::seed; use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate}; @@ -69,6 +69,31 @@ enum SourceKind { ModelWritten, } +#[derive(Clone, Copy, ValueEnum)] +enum ModelFamilyArg { + Anthropic, + OpenAi, + Google, + Alibaba, + Meta, + Mistral, + Unrecorded, +} + +impl From for ModelFamily { + fn from(f: ModelFamilyArg) -> Self { + match f { + ModelFamilyArg::Anthropic => ModelFamily::Anthropic, + ModelFamilyArg::OpenAi => ModelFamily::OpenAi, + ModelFamilyArg::Google => ModelFamily::Google, + ModelFamilyArg::Alibaba => ModelFamily::Alibaba, + ModelFamilyArg::Meta => ModelFamily::Meta, + ModelFamilyArg::Mistral => ModelFamily::Mistral, + ModelFamilyArg::Unrecorded => ModelFamily::Unrecorded, + } + } +} + #[derive(Args)] struct AddArgs { #[arg(long)] @@ -98,8 +123,11 @@ struct AddArgs { author: Option, #[arg(long)] edition_url: Option, + #[arg(long = "model-family", value_enum)] + family: Option, + /// The generator string the fixture or run wrote, verbatim. #[arg(long)] - model_family: Option, + generator: Option, #[arg(long)] prompt_digest: Option, #[arg(long)] @@ -130,7 +158,11 @@ impl AddArgs { edition_url: get(&self.edition_url, "edition-url")?, }, SourceKind::ModelWritten => Source::ModelWritten { - model_family: get(&self.model_family, "model-family")?, + family: self + .family + .map(ModelFamily::from) + .context("this source needs --model-family")?, + generator: self.generator.clone(), prompt_digest: self.prompt_digest.clone(), }, }) diff --git a/crates/weksa/src/store.rs b/crates/weksa/src/store.rs index 5bcb40c..3b55f4f 100644 --- a/crates/weksa/src/store.rs +++ b/crates/weksa/src/store.rs @@ -16,8 +16,8 @@ use crate::validate::{Refusal, validate}; /// A broken store rule, named. #[derive(Clone, Debug, PartialEq, Eq)] pub enum StoreRefusal { - /// S1: an existing item key re-added with different text. - S1ItemTextImmutable { key: String }, + /// S1: an existing item key re-added with any field different. + S1ItemImmutable { key: String }, /// S2: a revision that is not latest+1 with parent latest. S2RevisionChain { expected: u32, got: u32 }, /// S3: a revision after a frozen one. @@ -34,6 +34,8 @@ pub enum StoreRefusal { expected: String, got: String, }, + /// S6: an item's normalized text equals another key's. + S6TextHeld { key: String, held_by: String }, /// A `provenance.corpus_item` key names no item. UnknownCorpusItem { key: String }, /// A packet key already holds a different packet. @@ -151,8 +153,10 @@ impl CorpusStore { self.get(key)?.ok_or_else(|| unknown().into()) } - /// S1: an item key holds its text for good. S5: a correction stays in its - /// predecessor's group, so it cannot move across the split. + /// S1: an item key holds its whole item for good. S6: a text held under + /// one key (live or withdrawn) is not re-added under another unless the new + /// item supersedes that key. S5: a correction stays in its predecessor's + /// group, so it cannot move across the split. pub fn add(&mut self, item: CorpusItem) -> Result { if key_slug(&item.id).is_none() { return Err(malformed(&item.id, "item key is not item::")); @@ -164,10 +168,20 @@ impl CorpusStore { )); } if let Some(existing) = self.get(&item.id)? { - if existing.text == item.text { + if existing == item { return Ok(AddOutcome::Unchanged); } - return Err(StoreRefusal::S1ItemTextImmutable { key: item.id }.into()); + return Err(StoreRefusal::S1ItemImmutable { key: item.id }.into()); + } + let text = normalized(&item.text); + for held in self.all()? { + if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) { + return Err(StoreRefusal::S6TextHeld { + key: item.id, + held_by: held.id, + } + .into()); + } } if let Some(old) = &item.supersedes { let Some(old) = self.get(old)? else { @@ -202,6 +216,15 @@ impl CorpusStore { } } +/// Lowercased alphanumerics only, so case, spacing and punctuation variants of +/// a text compare equal. +fn normalized(text: &str) -> String { + text.chars() + .flat_map(char::to_lowercase) + .filter(|c| c.is_alphanumeric()) + .collect() +} + pub struct InventoryStore { cache: CultCache, } diff --git a/crates/weksa/tests/cli.rs b/crates/weksa/tests/cli.rs index 1d4f19a..6b13b77 100644 --- a/crates/weksa/tests/cli.rs +++ b/crates/weksa/tests/cli.rs @@ -65,7 +65,16 @@ fn seed_add_dump_split_withdraw() { "{}", out(&added) ); - let second = weksa(d.path(), &add); + // The same text under a new key is refused (S6); another text is not. + let copy = weksa(d.path(), &add); + assert!( + !copy.status.success() && out(©).contains("S6TextHeld"), + "{}", + out(©) + ); + let mut other = add.clone(); + other[5] = "Mind the other hatch."; + let second = weksa(d.path(), &other); assert!( out(&second).contains("Added item:nibu:2"), "{}", @@ -101,8 +110,6 @@ fn model_written_and_public_domain_sources_name_their_provenance() { "add", "--slug", "x", - "--text", - "t", "--speaker", "s", "--group", @@ -110,8 +117,13 @@ fn model_written_and_public_domain_sources_name_their_provenance() { "--admitted-on", "2026-10-29", ]; + // S6 refuses one text under two keys, so each run writes its own. + let runs = std::cell::Cell::new(0); let run = |extra: &[&str]| { + runs.set(runs.get() + 1); + let text = format!("line {}", runs.get()); let mut args = base.to_vec(); + args.extend_from_slice(&["--text", &text]); args.extend_from_slice(extra); weksa(d.path(), &args) }; @@ -121,8 +133,27 @@ fn model_written_and_public_domain_sources_name_their_provenance() { "{}", out(&no_family) ); - let model = run(&["--source", "model-written", "--model-family", "m"]); + let not_an_enum_value = run(&["--source", "model-written", "--model-family", "m"]); + assert!( + !not_an_enum_value.status.success(), + "{}", + out(¬_an_enum_value) + ); + let model = run(&[ + "--source", + "model-written", + "--model-family", + "open-ai", + "--generator", + "g", + ]); assert!(model.status.success(), "{}", out(&model)); + let dump = weksa(d.path(), &["corpus", "dump"]); + assert!( + out(&dump).contains("OpenAi") && out(&dump).contains("generator: Some(\"g\")"), + "{}", + out(&dump) + ); let no_work = run(&["--source", "public-domain", "--author", "a"]); assert!( !no_work.status.success() && out(&no_work).contains("--work"), diff --git a/crates/weksa/tests/common/mod.rs b/crates/weksa/tests/common/mod.rs index 7f7aea0..dfd8106 100644 --- a/crates/weksa/tests/common/mod.rs +++ b/crates/weksa/tests/common/mod.rs @@ -3,6 +3,7 @@ use std::sync::OnceLock; use tempfile::TempDir; +use weksa::corpus::ModelFamily; use weksa::inventory::{Change, seed}; use weksa::packet::*; use weksa::{ @@ -66,7 +67,8 @@ pub fn cast() -> Source { pub fn model() -> Source { Source::ModelWritten { - model_family: "m".into(), + family: ModelFamily::Unrecorded, + generator: None, prompt_digest: None, } } diff --git a/crates/weksa/tests/split_schema.rs b/crates/weksa/tests/split_schema.rs index d23737b..81e9cb1 100644 --- a/crates/weksa/tests/split_schema.rs +++ b/crates/weksa/tests/split_schema.rs @@ -1,4 +1,4 @@ -use weksa::corpus::{CorpusItem, Source}; +use weksa::corpus::{CorpusItem, ModelFamily, Source}; use weksa::split::fnv1a64; use weksa::{Split, SplitScheme}; @@ -26,7 +26,8 @@ fn cast() -> Source { fn model() -> Source { Source::ModelWritten { - model_family: "m".into(), + family: ModelFamily::Unrecorded, + generator: None, prompt_digest: None, } } diff --git a/crates/weksa/tests/stores.rs b/crates/weksa/tests/stores.rs index 5366191..c8aea2c 100644 --- a/crates/weksa/tests/stores.rs +++ b/crates/weksa/tests/stores.rs @@ -1,7 +1,7 @@ mod common; use common::*; use tempfile::TempDir; -use weksa::corpus::{CorpusItem, Source, Turn}; +use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn}; use weksa::inventory::{Change, seed}; use weksa::{ AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore, @@ -41,7 +41,7 @@ fn malformed(key: &str, why: &'static str) -> StoreRefusal { } #[test] -fn s1_item_text_immutable() { +fn s1_item_immutable() { let d = dir(); let path = d.path().join("c.cc"); let mut store = CorpusStore::open(&path).unwrap(); @@ -49,21 +49,164 @@ fn s1_item_text_immutable() { store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(), AddOutcome::Added ); - // The same text is idempotent, even from a reopened store. + // An equal item is idempotent, even from a reopened store. let mut store = CorpusStore::open(&path).unwrap(); assert_eq!( store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(), AddOutcome::Unchanged ); + let refused = || StoreRefusal::S1ItemImmutable { + key: "item:nibu:1".into(), + }; + // Changed text, and every other field, is refused; the stored item stays. + let base = item("item:nibu:1", "Mind the hatch."); + let mut variants = vec![item("item:nibu:1", "Mind the hatch!")]; + let mut v = base.clone(); + v.source = model(); + variants.push(v); + let mut v = base.clone(); + v.source = Source::OperatorWritten; + variants.push(v); + let mut v = base.clone(); + v.language = "en-GB".into(); + variants.push(v); + let mut v = base.clone(); + v.speaker = "brokkr".into(); + variants.push(v); + let mut v = base.clone(); + v.group = "scene-2".into(); + variants.push(v); + let mut v = base.clone(); + v.prior_turns.clear(); + variants.push(v); + let mut v = base.clone(); + v.admitted_on = "2026-10-30".into(); + variants.push(v); + let mut v = base.clone(); + v.supersedes = Some("item:nibu:9".into()); + variants.push(v); + for v in variants { + assert_eq!(refusal(store.add(v.clone())), refused(), "{v:?}"); + } + assert_eq!(store.get("item:nibu:1").unwrap(), Some(base.clone())); + // A withdrawn item is a different item: re-adding it live is refused too. + store.withdraw("item:nibu:1").unwrap(); + assert_eq!(refusal(store.add(base)), refused()); +} + +#[test] +fn corpus_keys_are_canonical() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + let long_slug = format!("item:a{}:1", "b".repeat(48)); + let not_canonical = [ + "item:a:01", + "item:a:+1", + "item:a:0", + "item:A:1", + "item:a-:1", + "item:-a:1", + "item:a:b:1", + "item::1", + "item:a:", + "item:a:\u{661}", + "item:a:4294967296", + &long_slug, + ]; + for (i, key) in not_canonical.into_iter().enumerate() { + assert_eq!( + refusal(store.add(item(key, &format!("line {i}")))), + malformed(key, "item key is not item::"), + "{key}" + ); + } + assert!(store.all().unwrap().is_empty()); + // The edges of the canonical form are admitted. + let longest = format!("item:{}:4294967295", "a".repeat(48)); + let canonical = ["item:a:1", "item:a-b:10", "item:7:2", "item:ab:9", &longest]; + for (i, key) in canonical.into_iter().enumerate() { + assert_eq!( + store.add(item(key, &format!("fine {i}"))).unwrap(), + AddOutcome::Added, + "{key}" + ); + } + // An alias of a stored key resolves to nothing. + let w = world(); + let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap(); + let mut p = good(); + p.provenance.corpus_item = Some("item:nibu:01".into()); assert_eq!( - refusal(store.add(item("item:nibu:1", "Mind the hatch!"))), - StoreRefusal::S1ItemTextImmutable { - key: "item:nibu:1".into() + refusal(packets.admit(p, &w.inventory, &w.corpus)), + StoreRefusal::UnknownCorpusItem { + key: "item:nibu:01".into() } ); +} + +#[test] +fn s6_text_held_by_another_key() { + let d = dir(); + let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap(); + store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(); + let elsewhere = |key: &str, text: &str| CorpusItem { + group: "scene-2".into(), + ..item(key, text) + }; + let held = |key: &str, by: &str| StoreRefusal::S6TextHeld { + key: key.into(), + held_by: by.into(), + }; + // The same text in another group, and in the same group, is refused. assert_eq!( - store.get("item:nibu:1").unwrap().unwrap().text, - "Mind the hatch." + refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."))), + held("item:nibu:2", "item:nibu:1") + ); + assert_eq!( + refusal(store.add(item("item:nibu:2", "Mind the hatch."))), + held("item:nibu:2", "item:nibu:1") + ); + // Case, spacing and punctuation variants are the same text. + for text in [ + "MIND THE HATCH", + "mind the\thatch...", + " Mind-the-hatch! ", + "Mind the hatch.\n", + ] { + assert_eq!( + refusal(store.add(elsewhere("item:nibu:2", text))), + held("item:nibu:2", "item:nibu:1"), + "{text:?}" + ); + } + store + .add(item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l")) + .unwrap(); + assert_eq!( + refusal(store.add(elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"))), + held("item:nibu:4", "item:nibu:3") + ); + // Text that differs in a letter or digit is a different text. + store.add(item("item:nibu:5", "Mind the hatch 2")).unwrap(); + store.add(item("item:nibu:6", "Mind the hatches.")).unwrap(); + // A withdrawn holder still holds. + store.withdraw("item:nibu:1").unwrap(); + assert_eq!( + refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"))), + held("item:nibu:2", "item:nibu:1") + ); + // Superseding a different key does not excuse a copy of another item. + let mut copy = item("item:nibu:8", "Mind the hatches!"); + copy.supersedes = Some("item:nibu:5".into()); + assert_eq!(refusal(store.add(copy)), held("item:nibu:8", "item:nibu:6")); + // A superseder of the holder, in the holder's group, is Added. + let mut fix = item("item:nibu:2", "mind the hatch"); + fix.supersedes = Some("item:nibu:1".into()); + assert_eq!(store.add(fix).unwrap(), AddOutcome::Added); + // The superseded holder's text stays held against a third key. + assert_eq!( + refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."))), + held("item:nibu:7", "item:nibu:1") ); } @@ -627,10 +770,43 @@ fn the_committed_stores_load_and_hold_clean_data() { ); } } - assert!( + let by_id = |id: &str| { items .iter() - .any(|i| matches!(i.source, Source::ModelWritten { .. })), - "model-written lines are labelled" + .find(|i| i.id == id) + .unwrap_or_else(|| panic!("{id}")) + }; + // brokkr:1 carries the generator string its fixture wrote, verbatim. + let fixture = std::fs::read_to_string( + root.join("examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml"), + ) + .unwrap(); + let written = fixture + .lines() + .find_map(|l| l.strip_prefix("generator: ")) + .expect("the fixture names its generator"); + assert_eq!(written, "codex-local-source-aware-draft"); + assert_eq!( + by_id("item:brokkr:1").source, + Source::ModelWritten { + family: ModelFamily::OpenAi, + generator: Some(written.into()), + prompt_digest: None + } ); + // Fixtures that record no author are Unrecorded, never CastAuthored. + let unrecorded = Source::ModelWritten { + family: ModelFamily::Unrecorded, + generator: None, + prompt_digest: None, + }; + assert_eq!(by_id("item:nibu:1").source, unrecorded); + assert_eq!(by_id("item:nibu:2").source, unrecorded); + assert!( + !items + .iter() + .any(|i| matches!(i.source, Source::CastAuthored { .. })), + "no item claims a recorded human author" + ); + assert_eq!(items.len(), 3); } diff --git a/data/corpus/corpus.cc b/data/corpus/corpus.cc index 803fc17..dd24f7b 100644 --- a/data/corpus/corpus.cc +++ b/data/corpus/corpus.cc @@ -1 +1 @@ -“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item“”­item:brokkr:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ“š­item:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice¬ModelWritten’¸local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ÷š«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001¬CastAuthored’¤nibuÙ