GameCult/Weksa
Watch
9
0
Fork
You've already forked Weksa
 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

packet-store-corpus (2b): truthful corpus labels with ModelFamily and verbatim generator, whole-item S1, canonical keys, S6 no copy under a new key (squash of weksa-lowering/packet-store-corpus 491536c..f9258d5; Soul s1)

Browse source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Eureka 2026-10-29 14:34:00 +01:00
commit c47e8e415d
10 changed files with 354 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

33
crates/weksa/src/corpus.rs
View file

@ -29,12 +29,28 @@ pub enum Source {
edition_url: String,
},
/// Labelled model output; never in the test split, never judged by its own family.
/// `family` is `Unrecorded` when nothing records one. `generator` is the
/// string the fixture or run wrote, kept verbatim.
ModelWritten {
model_family: String,
family: ModelFamily,
generator: Option<String>,
prompt_digest: Option<String>,
},
}
/// The family of the model that wrote a line.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub enum ModelFamily {
Anthropic,
OpenAi,
Google,
Alibaba,
Meta,
Mistral,
/// Nothing records which family wrote the line.
Unrecorded,
}
/// Key: `item:<source-slug>:<n>`.
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
#[cultcache(type = "weksa.corpus_item")]
@ -68,10 +84,19 @@ pub fn item_key(slug: &str, n: u32) -> String {
format!("item:{slug}:{n}")
}
/// The slug of an item key, if it is `item:<slug>:<n>`.
/// The slug of an item key, if it is canonical `item:<slug>:<n>`: a slug of
/// `[a-z0-9]` and inner `-` (at most 48 bytes, no edge `-`), and `n` a decimal
/// of at least 1 without sign or leading zero that fits `u32`.
pub fn key_slug(key: &str) -> Option<&str> {
let rest = key.strip_prefix("item:")?;
let (slug, n) = rest.rsplit_once(':')?;
n.parse::<u32>().ok()?;
(!slug.is_empty()).then_some(slug)
let canonical_n =
n.bytes().all(|b| b.is_ascii_digit()) && !n.starts_with('0') && n.parse::<u32>().is_ok();
let b = slug.as_bytes();
let slug_ok = (1..=48).contains(&b.len())
&& b.iter()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-')
&& b[0] != b'-'
&& b[b.len() - 1] != b'-';
(canonical_n && slug_ok).then_some(slug)
}

38
crates/weksa/src/main.rs
View file

@ -5,7 +5,7 @@ use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use clap::{Args, Parser, Subcommand, ValueEnum};
use weksa::corpus::{CorpusItem, Source, Turn};
use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn};
use weksa::inventory::seed;
use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate};
@ -69,6 +69,31 @@ enum SourceKind {
ModelWritten,
}
#[derive(Clone, Copy, ValueEnum)]
enum ModelFamilyArg {
Anthropic,
OpenAi,
Google,
Alibaba,
Meta,
Mistral,
Unrecorded,
}
impl From<ModelFamilyArg> for ModelFamily {
fn from(f: ModelFamilyArg) -> Self {
match f {
ModelFamilyArg::Anthropic => ModelFamily::Anthropic,
ModelFamilyArg::OpenAi => ModelFamily::OpenAi,
ModelFamilyArg::Google => ModelFamily::Google,
ModelFamilyArg::Alibaba => ModelFamily::Alibaba,
ModelFamilyArg::Meta => ModelFamily::Meta,
ModelFamilyArg::Mistral => ModelFamily::Mistral,
ModelFamilyArg::Unrecorded => ModelFamily::Unrecorded,
}
}
}
#[derive(Args)]
struct AddArgs {
#[arg(long)]
@ -98,8 +123,11 @@ struct AddArgs {
author: Option<String>,
#[arg(long)]
edition_url: Option<String>,
#[arg(long = "model-family", value_enum)]
family: Option<ModelFamilyArg>,
/// The generator string the fixture or run wrote, verbatim.
#[arg(long)]
model_family: Option<String>,
generator: Option<String>,
#[arg(long)]
prompt_digest: Option<String>,
#[arg(long)]
@ -130,7 +158,11 @@ impl AddArgs {
edition_url: get(&self.edition_url, "edition-url")?,
},
SourceKind::ModelWritten => Source::ModelWritten {
model_family: get(&self.model_family, "model-family")?,
family: self
.family
.map(ModelFamily::from)
.context("this source needs --model-family")?,
generator: self.generator.clone(),
prompt_digest: self.prompt_digest.clone(),
},
})

35
crates/weksa/src/store.rs
View file

@ -16,8 +16,8 @@ use crate::validate::{Refusal, validate};
/// A broken store rule, named.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum StoreRefusal {
/// S1: an existing item key re-added with different text.
S1ItemTextImmutable { key: String },
/// S1: an existing item key re-added with any field different.
S1ItemImmutable { key: String },
/// S2: a revision that is not latest+1 with parent latest.
S2RevisionChain { expected: u32, got: u32 },
/// S3: a revision after a frozen one.
@ -34,6 +34,8 @@ pub enum StoreRefusal {
expected: String,
got: String,
},
/// S6: an item's normalized text equals another key's.
S6TextHeld { key: String, held_by: String },
/// A `provenance.corpus_item` key names no item.
UnknownCorpusItem { key: String },
/// A packet key already holds a different packet.
@ -151,8 +153,10 @@ impl CorpusStore {
self.get(key)?.ok_or_else(|| unknown().into())
}
/// S1: an item key holds its text for good. S5: a correction stays in its
/// predecessor's group, so it cannot move across the split.
/// S1: an item key holds its whole item for good. S6: a text held under
/// one key (live or withdrawn) is not re-added under another unless the new
/// item supersedes that key. S5: a correction stays in its predecessor's
/// group, so it cannot move across the split.
pub fn add(&mut self, item: CorpusItem) -> Result<AddOutcome, StoreError> {
if key_slug(&item.id).is_none() {
return Err(malformed(&item.id, "item key is not item:<slug>:<n>"));
@ -164,10 +168,20 @@ impl CorpusStore {
));
}
if let Some(existing) = self.get(&item.id)? {
if existing.text == item.text {
if existing == item {
return Ok(AddOutcome::Unchanged);
}
return Err(StoreRefusal::S1ItemTextImmutable { key: item.id }.into());
return Err(StoreRefusal::S1ItemImmutable { key: item.id }.into());
}
let text = normalized(&item.text);
for held in self.all()? {
if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) {
return Err(StoreRefusal::S6TextHeld {
key: item.id,
held_by: held.id,
}
.into());
}
}
if let Some(old) = &item.supersedes {
let Some(old) = self.get(old)? else {
@ -202,6 +216,15 @@ impl CorpusStore {
}
}
/// Lowercased alphanumerics only, so case, spacing and punctuation variants of
/// a text compare equal.
fn normalized(text: &str) -> String {
text.chars()
.flat_map(char::to_lowercase)
.filter(|c| c.is_alphanumeric())
.collect()
}
pub struct InventoryStore {
cache: CultCache,
}

39
crates/weksa/tests/cli.rs
View file

@ -65,7 +65,16 @@ fn seed_add_dump_split_withdraw() {
"{}",
out(&added)
);
let second = weksa(d.path(), &add);
// The same text under a new key is refused (S6); another text is not.
let copy = weksa(d.path(), &add);
assert!(
!copy.status.success() && out(&copy).contains("S6TextHeld"),
"{}",
out(&copy)
);
let mut other = add.clone();
other[5] = "Mind the other hatch.";
let second = weksa(d.path(), &other);
assert!(
out(&second).contains("Added item:nibu:2"),
"{}",
@ -101,8 +110,6 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
"add",
"--slug",
"x",
"--text",
"t",
"--speaker",
"s",
"--group",
@ -110,8 +117,13 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
"--admitted-on",
"2026-10-29",
];
// S6 refuses one text under two keys, so each run writes its own.
let runs = std::cell::Cell::new(0);
let run = |extra: &[&str]| {
runs.set(runs.get() + 1);
let text = format!("line {}", runs.get());
let mut args = base.to_vec();
args.extend_from_slice(&["--text", &text]);
args.extend_from_slice(extra);
weksa(d.path(), &args)
};
@ -121,8 +133,27 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
"{}",
out(&no_family)
);
let model = run(&["--source", "model-written", "--model-family", "m"]);
let not_an_enum_value = run(&["--source", "model-written", "--model-family", "m"]);
assert!(
!not_an_enum_value.status.success(),
"{}",
out(&not_an_enum_value)
);
let model = run(&[
"--source",
"model-written",
"--model-family",
"open-ai",
"--generator",
"g",
]);
assert!(model.status.success(), "{}", out(&model));
let dump = weksa(d.path(), &["corpus", "dump"]);
assert!(
out(&dump).contains("OpenAi") && out(&dump).contains("generator: Some(\"g\")"),
"{}",
out(&dump)
);
let no_work = run(&["--source", "public-domain", "--author", "a"]);
assert!(
!no_work.status.success() && out(&no_work).contains("--work"),

4
crates/weksa/tests/common/mod.rs
View file

@ -3,6 +3,7 @@
use std::sync::OnceLock;
use tempfile::TempDir;
use weksa::corpus::ModelFamily;
use weksa::inventory::{Change, seed};
use weksa::packet::*;
use weksa::{
@ -66,7 +67,8 @@ pub fn cast() -> Source {
pub fn model() -> Source {
Source::ModelWritten {
model_family: "m".into(),
family: ModelFamily::Unrecorded,
generator: None,
prompt_digest: None,
}
}

5
crates/weksa/tests/split_schema.rs
View file

@ -1,4 +1,4 @@
use weksa::corpus::{CorpusItem, Source};
use weksa::corpus::{CorpusItem, ModelFamily, Source};
use weksa::split::fnv1a64;
use weksa::{Split, SplitScheme};
@ -26,7 +26,8 @@ fn cast() -> Source {
fn model() -> Source {
Source::ModelWritten {
model_family: "m".into(),
family: ModelFamily::Unrecorded,
generator: None,
prompt_digest: None,
}
}

200
crates/weksa/tests/stores.rs
View file

@ -1,7 +1,7 @@
mod common;
use common::*;
use tempfile::TempDir;
use weksa::corpus::{CorpusItem, Source, Turn};
use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn};
use weksa::inventory::{Change, seed};
use weksa::{
AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore,
@ -41,7 +41,7 @@ fn malformed(key: &str, why: &'static str) -> StoreRefusal {
}
#[test]
fn s1_item_text_immutable() {
fn s1_item_immutable() {
let d = dir();
let path = d.path().join("c.cc");
let mut store = CorpusStore::open(&path).unwrap();
@ -49,21 +49,164 @@ fn s1_item_text_immutable() {
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
AddOutcome::Added
);
// The same text is idempotent, even from a reopened store.
// An equal item is idempotent, even from a reopened store.
let mut store = CorpusStore::open(&path).unwrap();
assert_eq!(
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
AddOutcome::Unchanged
);
let refused = || StoreRefusal::S1ItemImmutable {
key: "item:nibu:1".into(),
};
// Changed text, and every other field, is refused; the stored item stays.
let base = item("item:nibu:1", "Mind the hatch.");
let mut variants = vec![item("item:nibu:1", "Mind the hatch!")];
let mut v = base.clone();
v.source = model();
variants.push(v);
let mut v = base.clone();
v.source = Source::OperatorWritten;
variants.push(v);
let mut v = base.clone();
v.language = "en-GB".into();
variants.push(v);
let mut v = base.clone();
v.speaker = "brokkr".into();
variants.push(v);
let mut v = base.clone();
v.group = "scene-2".into();
variants.push(v);
let mut v = base.clone();
v.prior_turns.clear();
variants.push(v);
let mut v = base.clone();
v.admitted_on = "2026-10-30".into();
variants.push(v);
let mut v = base.clone();
v.supersedes = Some("item:nibu:9".into());
variants.push(v);
for v in variants {
assert_eq!(refusal(store.add(v.clone())), refused(), "{v:?}");
}
assert_eq!(store.get("item:nibu:1").unwrap(), Some(base.clone()));
// A withdrawn item is a different item: re-adding it live is refused too.
store.withdraw("item:nibu:1").unwrap();
assert_eq!(refusal(store.add(base)), refused());
}
#[test]
fn corpus_keys_are_canonical() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
let long_slug = format!("item:a{}:1", "b".repeat(48));
let not_canonical = [
"item:a:01",
"item:a:+1",
"item:a:0",
"item:A:1",
"item:a-:1",
"item:-a:1",
"item:a:b:1",
"item::1",
"item:a:",
"item:a:\u{661}",
"item:a:4294967296",
&long_slug,
];
for (i, key) in not_canonical.into_iter().enumerate() {
assert_eq!(
refusal(store.add(item("item:nibu:1", "Mind the hatch!"))),
StoreRefusal::S1ItemTextImmutable {
key: "item:nibu:1".into()
refusal(store.add(item(key, &format!("line {i}")))),
malformed(key, "item key is not item:<slug>:<n>"),
"{key}"
);
}
assert!(store.all().unwrap().is_empty());
// The edges of the canonical form are admitted.
let longest = format!("item:{}:4294967295", "a".repeat(48));
let canonical = ["item:a:1", "item:a-b:10", "item:7:2", "item:ab:9", &longest];
for (i, key) in canonical.into_iter().enumerate() {
assert_eq!(
store.add(item(key, &format!("fine {i}"))).unwrap(),
AddOutcome::Added,
"{key}"
);
}
// An alias of a stored key resolves to nothing.
let w = world();
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
let mut p = good();
p.provenance.corpus_item = Some("item:nibu:01".into());
assert_eq!(
refusal(packets.admit(p, &w.inventory, &w.corpus)),
StoreRefusal::UnknownCorpusItem {
key: "item:nibu:01".into()
}
);
}
#[test]
fn s6_text_held_by_another_key() {
let d = dir();
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap();
let elsewhere = |key: &str, text: &str| CorpusItem {
group: "scene-2".into(),
..item(key, text)
};
let held = |key: &str, by: &str| StoreRefusal::S6TextHeld {
key: key.into(),
held_by: by.into(),
};
// The same text in another group, and in the same group, is refused.
assert_eq!(
store.get("item:nibu:1").unwrap().unwrap().text,
"Mind the hatch."
refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."))),
held("item:nibu:2", "item:nibu:1")
);
assert_eq!(
refusal(store.add(item("item:nibu:2", "Mind the hatch."))),
held("item:nibu:2", "item:nibu:1")
);
// Case, spacing and punctuation variants are the same text.
for text in [
"MIND THE HATCH",
"mind the\thatch...",
" Mind-the-hatch! ",
"Mind the hatch.\n",
] {
assert_eq!(
refusal(store.add(elsewhere("item:nibu:2", text))),
held("item:nibu:2", "item:nibu:1"),
"{text:?}"
);
}
store
.add(item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l"))
.unwrap();
assert_eq!(
refusal(store.add(elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"))),
held("item:nibu:4", "item:nibu:3")
);
// Text that differs in a letter or digit is a different text.
store.add(item("item:nibu:5", "Mind the hatch 2")).unwrap();
store.add(item("item:nibu:6", "Mind the hatches.")).unwrap();
// A withdrawn holder still holds.
store.withdraw("item:nibu:1").unwrap();
assert_eq!(
refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"))),
held("item:nibu:2", "item:nibu:1")
);
// Superseding a different key does not excuse a copy of another item.
let mut copy = item("item:nibu:8", "Mind the hatches!");
copy.supersedes = Some("item:nibu:5".into());
assert_eq!(refusal(store.add(copy)), held("item:nibu:8", "item:nibu:6"));
// A superseder of the holder, in the holder's group, is Added.
let mut fix = item("item:nibu:2", "mind the hatch");
fix.supersedes = Some("item:nibu:1".into());
assert_eq!(store.add(fix).unwrap(), AddOutcome::Added);
// The superseded holder's text stays held against a third key.
assert_eq!(
refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."))),
held("item:nibu:7", "item:nibu:1")
);
}
@ -627,10 +770,43 @@ fn the_committed_stores_load_and_hold_clean_data() {
);
}
}
assert!(
let by_id = |id: &str| {
items
.iter()
.any(|i| matches!(i.source, Source::ModelWritten { .. })),
"model-written lines are labelled"
);
.find(|i| i.id == id)
.unwrap_or_else(|| panic!("{id}"))
};
// brokkr:1 carries the generator string its fixture wrote, verbatim.
let fixture = std::fs::read_to_string(
root.join("examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml"),
)
.unwrap();
let written = fixture
.lines()
.find_map(|l| l.strip_prefix("generator: "))
.expect("the fixture names its generator");
assert_eq!(written, "codex-local-source-aware-draft");
assert_eq!(
by_id("item:brokkr:1").source,
Source::ModelWritten {
family: ModelFamily::OpenAi,
generator: Some(written.into()),
prompt_digest: None
}
);
// Fixtures that record no author are Unrecorded, never CastAuthored.
let unrecorded = Source::ModelWritten {
family: ModelFamily::Unrecorded,
generator: None,
prompt_digest: None,
};
assert_eq!(by_id("item:nibu:1").source, unrecorded);
assert_eq!(by_id("item:nibu:2").source, unrecorded);
assert!(
!items
.iter()
.any(|i| matches!(i.source, Source::CastAuthored { .. })),
"no item claims a recorded human author"
);
assert_eq!(items.len(), 3);
}

2
data/corpus/corpus.cc
View file

@ -1 +1 @@
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”­item:brokkr:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ“š­item:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten’¸local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ÷š«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬CastAuthored’¤nibuÙ<examples/english-lowering/nibu-proof-001/lowered-output.yamlª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T09:44:22ZÄíš«item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten’´unrecorded-agent-runÀª2026-10-29ÀÂ
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”­item:brokkr:1±weksa.corpus_item´2026-10-29T13:16:54ZÄ š­item:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten“¦OpenAi¾codex-local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T13:16:54ZÄÁš«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T13:16:54ZÄäš«item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29ÀÂ

2
examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml
View file

@ -1,6 +1,6 @@
schema_version: weksa.persona_lowering_output.v0
fixture_id: brokkr-edda-voice
generator: local-source-aware-draft
generator: codex-local-source-aware-draft
items:
- id: command-without-receipt
utterance: "The command has no receipt. It is not truth."

35
schemas/cultnet/weksa.corpus_item.schema.json
View file

@ -55,6 +55,27 @@
"withdrawn"
],
"$defs": {
"ModelFamily": {
"description": "The family of the model that wrote a line.",
"oneOf": [
{
"type": "string",
"enum": [
"Anthropic",
"OpenAi",
"Google",
"Alibaba",
"Meta",
"Mistral"
]
},
{
"description": "Nothing records which family wrote the line.",
"type": "string",
"const": "Unrecorded"
}
]
},
"Source": {
"oneOf": [
{
@ -137,14 +158,20 @@
]
},
{
"description": "Labelled model output; never in the test split, never judged by its own family.",
"description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.",
"type": "object",
"properties": {
"ModelWritten": {
"type": "object",
"properties": {
"model_family": {
"type": "string"
"family": {
"$ref": "#/$defs/ModelFamily"
},
"generator": {
"type": [
"string",
"null"
]
},
"prompt_digest": {
"type": [
@ -154,7 +181,7 @@
}
},
"required": [
"model_family"
"family"
]
}
},