packet-store-corpus (2b): truthful corpus labels with ModelFamily and verbatim generator, whole-item S1, canonical keys, S6 no copy under a new key (squash of weksa-lowering/packet-store-corpus 491536c..f9258d5; Soul s1)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
491536cb30
commit
c47e8e415d
10 changed files with 354 additions and 37 deletions
|
|
@ -29,12 +29,28 @@ pub enum Source {
|
||||||
edition_url: String,
|
edition_url: String,
|
||||||
},
|
},
|
||||||
/// Labelled model output; never in the test split, never judged by its own family.
|
/// Labelled model output; never in the test split, never judged by its own family.
|
||||||
|
/// `family` is `Unrecorded` when nothing records one. `generator` is the
|
||||||
|
/// string the fixture or run wrote, kept verbatim.
|
||||||
ModelWritten {
|
ModelWritten {
|
||||||
model_family: String,
|
family: ModelFamily,
|
||||||
|
generator: Option<String>,
|
||||||
prompt_digest: Option<String>,
|
prompt_digest: Option<String>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The family of the model that wrote a line.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub enum ModelFamily {
|
||||||
|
Anthropic,
|
||||||
|
OpenAi,
|
||||||
|
Google,
|
||||||
|
Alibaba,
|
||||||
|
Meta,
|
||||||
|
Mistral,
|
||||||
|
/// Nothing records which family wrote the line.
|
||||||
|
Unrecorded,
|
||||||
|
}
|
||||||
|
|
||||||
/// Key: `item:<source-slug>:<n>`.
|
/// Key: `item:<source-slug>:<n>`.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
#[derive(Clone, Debug, PartialEq, Eq, JsonSchema, DatabaseEntry)]
|
||||||
#[cultcache(type = "weksa.corpus_item")]
|
#[cultcache(type = "weksa.corpus_item")]
|
||||||
|
|
@ -68,10 +84,19 @@ pub fn item_key(slug: &str, n: u32) -> String {
|
||||||
format!("item:{slug}:{n}")
|
format!("item:{slug}:{n}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The slug of an item key, if it is `item:<slug>:<n>`.
|
/// The slug of an item key, if it is canonical `item:<slug>:<n>`: a slug of
|
||||||
|
/// `[a-z0-9]` and inner `-` (at most 48 bytes, no edge `-`), and `n` a decimal
|
||||||
|
/// of at least 1 without sign or leading zero that fits `u32`.
|
||||||
pub fn key_slug(key: &str) -> Option<&str> {
|
pub fn key_slug(key: &str) -> Option<&str> {
|
||||||
let rest = key.strip_prefix("item:")?;
|
let rest = key.strip_prefix("item:")?;
|
||||||
let (slug, n) = rest.rsplit_once(':')?;
|
let (slug, n) = rest.rsplit_once(':')?;
|
||||||
n.parse::<u32>().ok()?;
|
let canonical_n =
|
||||||
(!slug.is_empty()).then_some(slug)
|
n.bytes().all(|b| b.is_ascii_digit()) && !n.starts_with('0') && n.parse::<u32>().is_ok();
|
||||||
|
let b = slug.as_bytes();
|
||||||
|
let slug_ok = (1..=48).contains(&b.len())
|
||||||
|
&& b.iter()
|
||||||
|
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-')
|
||||||
|
&& b[0] != b'-'
|
||||||
|
&& b[b.len() - 1] != b'-';
|
||||||
|
(canonical_n && slug_ok).then_some(slug)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ use std::path::PathBuf;
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||||
use weksa::corpus::{CorpusItem, Source, Turn};
|
use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn};
|
||||||
use weksa::inventory::seed;
|
use weksa::inventory::seed;
|
||||||
use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate};
|
use weksa::{CorpusStore, InventoryStore, PacketStore, SplitScheme, StoreError, validate};
|
||||||
|
|
||||||
|
|
@ -69,6 +69,31 @@ enum SourceKind {
|
||||||
ModelWritten,
|
ModelWritten,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, ValueEnum)]
|
||||||
|
enum ModelFamilyArg {
|
||||||
|
Anthropic,
|
||||||
|
OpenAi,
|
||||||
|
Google,
|
||||||
|
Alibaba,
|
||||||
|
Meta,
|
||||||
|
Mistral,
|
||||||
|
Unrecorded,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ModelFamilyArg> for ModelFamily {
|
||||||
|
fn from(f: ModelFamilyArg) -> Self {
|
||||||
|
match f {
|
||||||
|
ModelFamilyArg::Anthropic => ModelFamily::Anthropic,
|
||||||
|
ModelFamilyArg::OpenAi => ModelFamily::OpenAi,
|
||||||
|
ModelFamilyArg::Google => ModelFamily::Google,
|
||||||
|
ModelFamilyArg::Alibaba => ModelFamily::Alibaba,
|
||||||
|
ModelFamilyArg::Meta => ModelFamily::Meta,
|
||||||
|
ModelFamilyArg::Mistral => ModelFamily::Mistral,
|
||||||
|
ModelFamilyArg::Unrecorded => ModelFamily::Unrecorded,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Args)]
|
#[derive(Args)]
|
||||||
struct AddArgs {
|
struct AddArgs {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
|
|
@ -98,8 +123,11 @@ struct AddArgs {
|
||||||
author: Option<String>,
|
author: Option<String>,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
edition_url: Option<String>,
|
edition_url: Option<String>,
|
||||||
|
#[arg(long = "model-family", value_enum)]
|
||||||
|
family: Option<ModelFamilyArg>,
|
||||||
|
/// The generator string the fixture or run wrote, verbatim.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
model_family: Option<String>,
|
generator: Option<String>,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
prompt_digest: Option<String>,
|
prompt_digest: Option<String>,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
|
|
@ -130,7 +158,11 @@ impl AddArgs {
|
||||||
edition_url: get(&self.edition_url, "edition-url")?,
|
edition_url: get(&self.edition_url, "edition-url")?,
|
||||||
},
|
},
|
||||||
SourceKind::ModelWritten => Source::ModelWritten {
|
SourceKind::ModelWritten => Source::ModelWritten {
|
||||||
model_family: get(&self.model_family, "model-family")?,
|
family: self
|
||||||
|
.family
|
||||||
|
.map(ModelFamily::from)
|
||||||
|
.context("this source needs --model-family")?,
|
||||||
|
generator: self.generator.clone(),
|
||||||
prompt_digest: self.prompt_digest.clone(),
|
prompt_digest: self.prompt_digest.clone(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -16,8 +16,8 @@ use crate::validate::{Refusal, validate};
|
||||||
/// A broken store rule, named.
|
/// A broken store rule, named.
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum StoreRefusal {
|
pub enum StoreRefusal {
|
||||||
/// S1: an existing item key re-added with different text.
|
/// S1: an existing item key re-added with any field different.
|
||||||
S1ItemTextImmutable { key: String },
|
S1ItemImmutable { key: String },
|
||||||
/// S2: a revision that is not latest+1 with parent latest.
|
/// S2: a revision that is not latest+1 with parent latest.
|
||||||
S2RevisionChain { expected: u32, got: u32 },
|
S2RevisionChain { expected: u32, got: u32 },
|
||||||
/// S3: a revision after a frozen one.
|
/// S3: a revision after a frozen one.
|
||||||
|
|
@ -34,6 +34,8 @@ pub enum StoreRefusal {
|
||||||
expected: String,
|
expected: String,
|
||||||
got: String,
|
got: String,
|
||||||
},
|
},
|
||||||
|
/// S6: an item's normalized text equals another key's.
|
||||||
|
S6TextHeld { key: String, held_by: String },
|
||||||
/// A `provenance.corpus_item` key names no item.
|
/// A `provenance.corpus_item` key names no item.
|
||||||
UnknownCorpusItem { key: String },
|
UnknownCorpusItem { key: String },
|
||||||
/// A packet key already holds a different packet.
|
/// A packet key already holds a different packet.
|
||||||
|
|
@ -151,8 +153,10 @@ impl CorpusStore {
|
||||||
self.get(key)?.ok_or_else(|| unknown().into())
|
self.get(key)?.ok_or_else(|| unknown().into())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// S1: an item key holds its text for good. S5: a correction stays in its
|
/// S1: an item key holds its whole item for good. S6: a text held under
|
||||||
/// predecessor's group, so it cannot move across the split.
|
/// one key (live or withdrawn) is not re-added under another unless the new
|
||||||
|
/// item supersedes that key. S5: a correction stays in its predecessor's
|
||||||
|
/// group, so it cannot move across the split.
|
||||||
pub fn add(&mut self, item: CorpusItem) -> Result<AddOutcome, StoreError> {
|
pub fn add(&mut self, item: CorpusItem) -> Result<AddOutcome, StoreError> {
|
||||||
if key_slug(&item.id).is_none() {
|
if key_slug(&item.id).is_none() {
|
||||||
return Err(malformed(&item.id, "item key is not item:<slug>:<n>"));
|
return Err(malformed(&item.id, "item key is not item:<slug>:<n>"));
|
||||||
|
|
@ -164,10 +168,20 @@ impl CorpusStore {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if let Some(existing) = self.get(&item.id)? {
|
if let Some(existing) = self.get(&item.id)? {
|
||||||
if existing.text == item.text {
|
if existing == item {
|
||||||
return Ok(AddOutcome::Unchanged);
|
return Ok(AddOutcome::Unchanged);
|
||||||
}
|
}
|
||||||
return Err(StoreRefusal::S1ItemTextImmutable { key: item.id }.into());
|
return Err(StoreRefusal::S1ItemImmutable { key: item.id }.into());
|
||||||
|
}
|
||||||
|
let text = normalized(&item.text);
|
||||||
|
for held in self.all()? {
|
||||||
|
if normalized(&held.text) == text && item.supersedes.as_deref() != Some(&held.id) {
|
||||||
|
return Err(StoreRefusal::S6TextHeld {
|
||||||
|
key: item.id,
|
||||||
|
held_by: held.id,
|
||||||
|
}
|
||||||
|
.into());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if let Some(old) = &item.supersedes {
|
if let Some(old) = &item.supersedes {
|
||||||
let Some(old) = self.get(old)? else {
|
let Some(old) = self.get(old)? else {
|
||||||
|
|
@ -202,6 +216,15 @@ impl CorpusStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Lowercased alphanumerics only, so case, spacing and punctuation variants of
|
||||||
|
/// a text compare equal.
|
||||||
|
fn normalized(text: &str) -> String {
|
||||||
|
text.chars()
|
||||||
|
.flat_map(char::to_lowercase)
|
||||||
|
.filter(|c| c.is_alphanumeric())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
pub struct InventoryStore {
|
pub struct InventoryStore {
|
||||||
cache: CultCache,
|
cache: CultCache,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,16 @@ fn seed_add_dump_split_withdraw() {
|
||||||
"{}",
|
"{}",
|
||||||
out(&added)
|
out(&added)
|
||||||
);
|
);
|
||||||
let second = weksa(d.path(), &add);
|
// The same text under a new key is refused (S6); another text is not.
|
||||||
|
let copy = weksa(d.path(), &add);
|
||||||
|
assert!(
|
||||||
|
!copy.status.success() && out(©).contains("S6TextHeld"),
|
||||||
|
"{}",
|
||||||
|
out(©)
|
||||||
|
);
|
||||||
|
let mut other = add.clone();
|
||||||
|
other[5] = "Mind the other hatch.";
|
||||||
|
let second = weksa(d.path(), &other);
|
||||||
assert!(
|
assert!(
|
||||||
out(&second).contains("Added item:nibu:2"),
|
out(&second).contains("Added item:nibu:2"),
|
||||||
"{}",
|
"{}",
|
||||||
|
|
@ -101,8 +110,6 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
|
||||||
"add",
|
"add",
|
||||||
"--slug",
|
"--slug",
|
||||||
"x",
|
"x",
|
||||||
"--text",
|
|
||||||
"t",
|
|
||||||
"--speaker",
|
"--speaker",
|
||||||
"s",
|
"s",
|
||||||
"--group",
|
"--group",
|
||||||
|
|
@ -110,8 +117,13 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
|
||||||
"--admitted-on",
|
"--admitted-on",
|
||||||
"2026-10-29",
|
"2026-10-29",
|
||||||
];
|
];
|
||||||
|
// S6 refuses one text under two keys, so each run writes its own.
|
||||||
|
let runs = std::cell::Cell::new(0);
|
||||||
let run = |extra: &[&str]| {
|
let run = |extra: &[&str]| {
|
||||||
|
runs.set(runs.get() + 1);
|
||||||
|
let text = format!("line {}", runs.get());
|
||||||
let mut args = base.to_vec();
|
let mut args = base.to_vec();
|
||||||
|
args.extend_from_slice(&["--text", &text]);
|
||||||
args.extend_from_slice(extra);
|
args.extend_from_slice(extra);
|
||||||
weksa(d.path(), &args)
|
weksa(d.path(), &args)
|
||||||
};
|
};
|
||||||
|
|
@ -121,8 +133,27 @@ fn model_written_and_public_domain_sources_name_their_provenance() {
|
||||||
"{}",
|
"{}",
|
||||||
out(&no_family)
|
out(&no_family)
|
||||||
);
|
);
|
||||||
let model = run(&["--source", "model-written", "--model-family", "m"]);
|
let not_an_enum_value = run(&["--source", "model-written", "--model-family", "m"]);
|
||||||
|
assert!(
|
||||||
|
!not_an_enum_value.status.success(),
|
||||||
|
"{}",
|
||||||
|
out(¬_an_enum_value)
|
||||||
|
);
|
||||||
|
let model = run(&[
|
||||||
|
"--source",
|
||||||
|
"model-written",
|
||||||
|
"--model-family",
|
||||||
|
"open-ai",
|
||||||
|
"--generator",
|
||||||
|
"g",
|
||||||
|
]);
|
||||||
assert!(model.status.success(), "{}", out(&model));
|
assert!(model.status.success(), "{}", out(&model));
|
||||||
|
let dump = weksa(d.path(), &["corpus", "dump"]);
|
||||||
|
assert!(
|
||||||
|
out(&dump).contains("OpenAi") && out(&dump).contains("generator: Some(\"g\")"),
|
||||||
|
"{}",
|
||||||
|
out(&dump)
|
||||||
|
);
|
||||||
let no_work = run(&["--source", "public-domain", "--author", "a"]);
|
let no_work = run(&["--source", "public-domain", "--author", "a"]);
|
||||||
assert!(
|
assert!(
|
||||||
!no_work.status.success() && out(&no_work).contains("--work"),
|
!no_work.status.success() && out(&no_work).contains("--work"),
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
use weksa::corpus::ModelFamily;
|
||||||
use weksa::inventory::{Change, seed};
|
use weksa::inventory::{Change, seed};
|
||||||
use weksa::packet::*;
|
use weksa::packet::*;
|
||||||
use weksa::{
|
use weksa::{
|
||||||
|
|
@ -66,7 +67,8 @@ pub fn cast() -> Source {
|
||||||
|
|
||||||
pub fn model() -> Source {
|
pub fn model() -> Source {
|
||||||
Source::ModelWritten {
|
Source::ModelWritten {
|
||||||
model_family: "m".into(),
|
family: ModelFamily::Unrecorded,
|
||||||
|
generator: None,
|
||||||
prompt_digest: None,
|
prompt_digest: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use weksa::corpus::{CorpusItem, Source};
|
use weksa::corpus::{CorpusItem, ModelFamily, Source};
|
||||||
use weksa::split::fnv1a64;
|
use weksa::split::fnv1a64;
|
||||||
use weksa::{Split, SplitScheme};
|
use weksa::{Split, SplitScheme};
|
||||||
|
|
||||||
|
|
@ -26,7 +26,8 @@ fn cast() -> Source {
|
||||||
|
|
||||||
fn model() -> Source {
|
fn model() -> Source {
|
||||||
Source::ModelWritten {
|
Source::ModelWritten {
|
||||||
model_family: "m".into(),
|
family: ModelFamily::Unrecorded,
|
||||||
|
generator: None,
|
||||||
prompt_digest: None,
|
prompt_digest: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
mod common;
|
mod common;
|
||||||
use common::*;
|
use common::*;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use weksa::corpus::{CorpusItem, Source, Turn};
|
use weksa::corpus::{CorpusItem, ModelFamily, Source, Turn};
|
||||||
use weksa::inventory::{Change, seed};
|
use weksa::inventory::{Change, seed};
|
||||||
use weksa::{
|
use weksa::{
|
||||||
AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore,
|
AddOutcome, CorpusStore, Family, InventoryEntry, InventoryRevision, InventoryStore,
|
||||||
|
|
@ -41,7 +41,7 @@ fn malformed(key: &str, why: &'static str) -> StoreRefusal {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn s1_item_text_immutable() {
|
fn s1_item_immutable() {
|
||||||
let d = dir();
|
let d = dir();
|
||||||
let path = d.path().join("c.cc");
|
let path = d.path().join("c.cc");
|
||||||
let mut store = CorpusStore::open(&path).unwrap();
|
let mut store = CorpusStore::open(&path).unwrap();
|
||||||
|
|
@ -49,21 +49,164 @@ fn s1_item_text_immutable() {
|
||||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
||||||
AddOutcome::Added
|
AddOutcome::Added
|
||||||
);
|
);
|
||||||
// The same text is idempotent, even from a reopened store.
|
// An equal item is idempotent, even from a reopened store.
|
||||||
let mut store = CorpusStore::open(&path).unwrap();
|
let mut store = CorpusStore::open(&path).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap(),
|
||||||
AddOutcome::Unchanged
|
AddOutcome::Unchanged
|
||||||
);
|
);
|
||||||
|
let refused = || StoreRefusal::S1ItemImmutable {
|
||||||
|
key: "item:nibu:1".into(),
|
||||||
|
};
|
||||||
|
// Changed text, and every other field, is refused; the stored item stays.
|
||||||
|
let base = item("item:nibu:1", "Mind the hatch.");
|
||||||
|
let mut variants = vec![item("item:nibu:1", "Mind the hatch!")];
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.source = model();
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.source = Source::OperatorWritten;
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.language = "en-GB".into();
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.speaker = "brokkr".into();
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.group = "scene-2".into();
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.prior_turns.clear();
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.admitted_on = "2026-10-30".into();
|
||||||
|
variants.push(v);
|
||||||
|
let mut v = base.clone();
|
||||||
|
v.supersedes = Some("item:nibu:9".into());
|
||||||
|
variants.push(v);
|
||||||
|
for v in variants {
|
||||||
|
assert_eq!(refusal(store.add(v.clone())), refused(), "{v:?}");
|
||||||
|
}
|
||||||
|
assert_eq!(store.get("item:nibu:1").unwrap(), Some(base.clone()));
|
||||||
|
// A withdrawn item is a different item: re-adding it live is refused too.
|
||||||
|
store.withdraw("item:nibu:1").unwrap();
|
||||||
|
assert_eq!(refusal(store.add(base)), refused());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn corpus_keys_are_canonical() {
|
||||||
|
let d = dir();
|
||||||
|
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||||
|
let long_slug = format!("item:a{}:1", "b".repeat(48));
|
||||||
|
let not_canonical = [
|
||||||
|
"item:a:01",
|
||||||
|
"item:a:+1",
|
||||||
|
"item:a:0",
|
||||||
|
"item:A:1",
|
||||||
|
"item:a-:1",
|
||||||
|
"item:-a:1",
|
||||||
|
"item:a:b:1",
|
||||||
|
"item::1",
|
||||||
|
"item:a:",
|
||||||
|
"item:a:\u{661}",
|
||||||
|
"item:a:4294967296",
|
||||||
|
&long_slug,
|
||||||
|
];
|
||||||
|
for (i, key) in not_canonical.into_iter().enumerate() {
|
||||||
|
assert_eq!(
|
||||||
|
refusal(store.add(item(key, &format!("line {i}")))),
|
||||||
|
malformed(key, "item key is not item:<slug>:<n>"),
|
||||||
|
"{key}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(store.all().unwrap().is_empty());
|
||||||
|
// The edges of the canonical form are admitted.
|
||||||
|
let longest = format!("item:{}:4294967295", "a".repeat(48));
|
||||||
|
let canonical = ["item:a:1", "item:a-b:10", "item:7:2", "item:ab:9", &longest];
|
||||||
|
for (i, key) in canonical.into_iter().enumerate() {
|
||||||
|
assert_eq!(
|
||||||
|
store.add(item(key, &format!("fine {i}"))).unwrap(),
|
||||||
|
AddOutcome::Added,
|
||||||
|
"{key}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// An alias of a stored key resolves to nothing.
|
||||||
|
let w = world();
|
||||||
|
let mut packets = PacketStore::open(&w.dir.path().join("p.cc")).unwrap();
|
||||||
|
let mut p = good();
|
||||||
|
p.provenance.corpus_item = Some("item:nibu:01".into());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
refusal(store.add(item("item:nibu:1", "Mind the hatch!"))),
|
refusal(packets.admit(p, &w.inventory, &w.corpus)),
|
||||||
StoreRefusal::S1ItemTextImmutable {
|
StoreRefusal::UnknownCorpusItem {
|
||||||
key: "item:nibu:1".into()
|
key: "item:nibu:01".into()
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn s6_text_held_by_another_key() {
|
||||||
|
let d = dir();
|
||||||
|
let mut store = CorpusStore::open(&d.path().join("c.cc")).unwrap();
|
||||||
|
store.add(item("item:nibu:1", "Mind the hatch.")).unwrap();
|
||||||
|
let elsewhere = |key: &str, text: &str| CorpusItem {
|
||||||
|
group: "scene-2".into(),
|
||||||
|
..item(key, text)
|
||||||
|
};
|
||||||
|
let held = |key: &str, by: &str| StoreRefusal::S6TextHeld {
|
||||||
|
key: key.into(),
|
||||||
|
held_by: by.into(),
|
||||||
|
};
|
||||||
|
// The same text in another group, and in the same group, is refused.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
store.get("item:nibu:1").unwrap().unwrap().text,
|
refusal(store.add(elsewhere("item:nibu:2", "Mind the hatch."))),
|
||||||
"Mind the hatch."
|
held("item:nibu:2", "item:nibu:1")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
refusal(store.add(item("item:nibu:2", "Mind the hatch."))),
|
||||||
|
held("item:nibu:2", "item:nibu:1")
|
||||||
|
);
|
||||||
|
// Case, spacing and punctuation variants are the same text.
|
||||||
|
for text in [
|
||||||
|
"MIND THE HATCH",
|
||||||
|
"mind the\thatch...",
|
||||||
|
" Mind-the-hatch! ",
|
||||||
|
"Mind the hatch.\n",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
refusal(store.add(elsewhere("item:nibu:2", text))),
|
||||||
|
held("item:nibu:2", "item:nibu:1"),
|
||||||
|
"{text:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
store
|
||||||
|
.add(item("item:nibu:3", "\u{c4}rger \u{fc}ber \u{d6}l"))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
refusal(store.add(elsewhere("item:nibu:4", "\u{e4}rger \u{dc}BER \u{f6}l"))),
|
||||||
|
held("item:nibu:4", "item:nibu:3")
|
||||||
|
);
|
||||||
|
// Text that differs in a letter or digit is a different text.
|
||||||
|
store.add(item("item:nibu:5", "Mind the hatch 2")).unwrap();
|
||||||
|
store.add(item("item:nibu:6", "Mind the hatches.")).unwrap();
|
||||||
|
// A withdrawn holder still holds.
|
||||||
|
store.withdraw("item:nibu:1").unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
refusal(store.add(elsewhere("item:nibu:2", "mind the hatch"))),
|
||||||
|
held("item:nibu:2", "item:nibu:1")
|
||||||
|
);
|
||||||
|
// Superseding a different key does not excuse a copy of another item.
|
||||||
|
let mut copy = item("item:nibu:8", "Mind the hatches!");
|
||||||
|
copy.supersedes = Some("item:nibu:5".into());
|
||||||
|
assert_eq!(refusal(store.add(copy)), held("item:nibu:8", "item:nibu:6"));
|
||||||
|
// A superseder of the holder, in the holder's group, is Added.
|
||||||
|
let mut fix = item("item:nibu:2", "mind the hatch");
|
||||||
|
fix.supersedes = Some("item:nibu:1".into());
|
||||||
|
assert_eq!(store.add(fix).unwrap(), AddOutcome::Added);
|
||||||
|
// The superseded holder's text stays held against a third key.
|
||||||
|
assert_eq!(
|
||||||
|
refusal(store.add(elsewhere("item:nibu:7", "Mind the hatch."))),
|
||||||
|
held("item:nibu:7", "item:nibu:1")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -627,10 +770,43 @@ fn the_committed_stores_load_and_hold_clean_data() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assert!(
|
let by_id = |id: &str| {
|
||||||
items
|
items
|
||||||
.iter()
|
.iter()
|
||||||
.any(|i| matches!(i.source, Source::ModelWritten { .. })),
|
.find(|i| i.id == id)
|
||||||
"model-written lines are labelled"
|
.unwrap_or_else(|| panic!("{id}"))
|
||||||
|
};
|
||||||
|
// brokkr:1 carries the generator string its fixture wrote, verbatim.
|
||||||
|
let fixture = std::fs::read_to_string(
|
||||||
|
root.join("examples/persona-lowering/brokkr-edda-voice/lowered-output.yaml"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let written = fixture
|
||||||
|
.lines()
|
||||||
|
.find_map(|l| l.strip_prefix("generator: "))
|
||||||
|
.expect("the fixture names its generator");
|
||||||
|
assert_eq!(written, "codex-local-source-aware-draft");
|
||||||
|
assert_eq!(
|
||||||
|
by_id("item:brokkr:1").source,
|
||||||
|
Source::ModelWritten {
|
||||||
|
family: ModelFamily::OpenAi,
|
||||||
|
generator: Some(written.into()),
|
||||||
|
prompt_digest: None
|
||||||
|
}
|
||||||
);
|
);
|
||||||
|
// Fixtures that record no author are Unrecorded, never CastAuthored.
|
||||||
|
let unrecorded = Source::ModelWritten {
|
||||||
|
family: ModelFamily::Unrecorded,
|
||||||
|
generator: None,
|
||||||
|
prompt_digest: None,
|
||||||
|
};
|
||||||
|
assert_eq!(by_id("item:nibu:1").source, unrecorded);
|
||||||
|
assert_eq!(by_id("item:nibu:2").source, unrecorded);
|
||||||
|
assert!(
|
||||||
|
!items
|
||||||
|
.iter()
|
||||||
|
.any(|i| matches!(i.source, Source::CastAuthored { .. })),
|
||||||
|
"no item claims a recorded human author"
|
||||||
|
);
|
||||||
|
assert_eq!(items.len(), 3);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”item:brokkr:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ“šitem:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten’¸local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T09:44:22ZÄ÷š«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬CastAuthored’¤nibuÙ<examples/english-lowering/nibu-proof-001/lowered-output.yamlª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T09:44:22ZÄíš«item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten’´unrecorded-agent-runÀª2026-10-29ÀÂ
|
“²cultcache.store.v1‘—±weksa.corpus_item±weksa.corpus_item´weksa.corpus_item.v1±weksa.corpus_itemÙV{"schemaName":"weksa.corpus_item","schemaVersion":"weksa.corpus_item.v1","members":[]}‘±weksa.corpus_item<EFBFBD>“”item:brokkr:1±weksa.corpus_item´2026-10-29T13:16:54ZÄ šitem:brokkr:1Ù,The command has no receipt. It is not truth.¥en-US¦brokkr±brokkr-edda-voice<EFBFBD><EFBFBD>¬ModelWritten“¦OpenAi¾codex-local-source-aware-draftÀª2026-10-29À”«item:nibu:1±weksa.corpus_item´2026-10-29T13:16:54ZÄÁš«item:nibu:1ÙnTake your hand off that control unless you want your nervous system to become my least useful circuit diagram.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29À”«item:nibu:2±weksa.corpus_item´2026-10-29T13:16:54ZÄäš«item:nibu:2Ù‘Do not touch that control. It is damaged; one touch may put you directly in harm's way. I am preventing an avoidable injury, not asking politely.¥en-US¤nibu®nibu-proof-001<EFBFBD><EFBFBD>¬ModelWritten“ªUnrecordedÀÀª2026-10-29ÀÂ
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
schema_version: weksa.persona_lowering_output.v0
|
schema_version: weksa.persona_lowering_output.v0
|
||||||
fixture_id: brokkr-edda-voice
|
fixture_id: brokkr-edda-voice
|
||||||
generator: local-source-aware-draft
|
generator: codex-local-source-aware-draft
|
||||||
items:
|
items:
|
||||||
- id: command-without-receipt
|
- id: command-without-receipt
|
||||||
utterance: "The command has no receipt. It is not truth."
|
utterance: "The command has no receipt. It is not truth."
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,27 @@
|
||||||
"withdrawn"
|
"withdrawn"
|
||||||
],
|
],
|
||||||
"$defs": {
|
"$defs": {
|
||||||
|
"ModelFamily": {
|
||||||
|
"description": "The family of the model that wrote a line.",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"Anthropic",
|
||||||
|
"OpenAi",
|
||||||
|
"Google",
|
||||||
|
"Alibaba",
|
||||||
|
"Meta",
|
||||||
|
"Mistral"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Nothing records which family wrote the line.",
|
||||||
|
"type": "string",
|
||||||
|
"const": "Unrecorded"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"Source": {
|
"Source": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
|
|
@ -137,14 +158,20 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Labelled model output; never in the test split, never judged by its own family.",
|
"description": "Labelled model output; never in the test split, never judged by its own family.\n`family` is `Unrecorded` when nothing records one. `generator` is the\nstring the fixture or run wrote, kept verbatim.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"ModelWritten": {
|
"ModelWritten": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"model_family": {
|
"family": {
|
||||||
"type": "string"
|
"$ref": "#/$defs/ModelFamily"
|
||||||
|
},
|
||||||
|
"generator": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"prompt_digest": {
|
"prompt_digest": {
|
||||||
"type": [
|
"type": [
|
||||||
|
|
@ -154,7 +181,7 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
"model_family"
|
"family"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue