From d02b56a69243ea9b390a6dd55e09826b4f5c729b Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 10:08:41 +0000 Subject: [PATCH 1/4] Unique-index vectors: pin holder identity by case (rows 23-24) Two holders of one value whose keys differ only by case are two holders; a case-folding dedupe would see one and miss the refusal. Co-Authored-By: Claude Sonnet 5.5 --- .../UniqueIndexVectorTests.cs | 8 +++++++- tests/vectors/unique-index/README.md | 13 +++++++------ .../vectors/unique-index/holder-case-store.cc | Bin 0 -> 943 bytes tests/vectors/unique-index/refusals.msgpack | Bin 2676 -> 2868 bytes 4 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 tests/vectors/unique-index/holder-case-store.cc diff --git a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs index 44aef30d..e3be8c8f 100644 --- a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs @@ -351,6 +351,7 @@ namespace GameCult.Caching.Tests private static readonly Rec[] RepullBefore = Items_(("item:a1", "A"), ("item:b1", "C")); private static readonly Rec[] RepullAfter = Items_(("item:a1", "A"), ("item:b1", "B"), ("item:b2", "B"), ("item:c1", "A")); private static readonly Rec[] KeyCaseRecords = Items_(("a", "S"), ("z", "S"), ("B", "T"), ("C", "T")); + private static readonly Rec[] HolderCaseRecords = Items_(("a", "S"), ("A", "S")); private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T")); private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T")); @@ -360,6 +361,7 @@ namespace GameCult.Caching.Tests ("repull-before.cc", RepullBefore), ("repull-after.cc", RepullAfter), ("alias-store.cc", AliasRecords), ("key-case-store.cc", KeyCaseRecords), ("key-punctuation-store.cc", KeyPunctuationRecords), ("surrogate-groups-store.cc", SurrogateGroupRecords), + ("holder-case-store.cc", HolderCaseRecords), }; private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku"); @@ -368,6 +370,7 @@ namespace GameCult.Caching.Tests private static readonly Refusal AliasRefusal = new("item:a", "item:b", Alias, "Sku"); private static readonly Refusal RepullRefusal = new("item:a1", "item:c1", Item, "Sku"); private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku"); + private static readonly Refusal HolderCaseRefusal = new("A", "a", Item, "Sku"); private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku"); private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku"); @@ -402,6 +405,9 @@ namespace GameCult.Caching.Tests new("load-key-case", new[] { Open("key-case-store.cc") }, KeyCaseRefusal), new("load-key-punctuation", new[] { Open("key-punctuation-store.cc") }, KeyPunctuationRefusal), new("load-surrogate-groups", new[] { Open("surrogate-groups-store.cc") }, SurrogateGroupRefusal), + // Rows 23-24: two holders of one value that differ only by case are two holders; a case-folding dedupe would see one. + new("batch-holders-by-case", new[] { Open(), Commit(HolderCaseRecords) }, HolderCaseRefusal), + new("load-holders-by-case", new[] { Open("holder-case-store.cc") }, HolderCaseRefusal), }; private static byte[] RefusalStoreBytes(Rec[] records) @@ -589,7 +595,7 @@ namespace GameCult.Caching.Tests } } - Assert.That(walked, Is.EqualTo(22)); + Assert.That(walked, Is.EqualTo(24)); } } } diff --git a/tests/vectors/unique-index/README.md b/tests/vectors/unique-index/README.md index ac4a40cd..621ebe36 100644 --- a/tests/vectors/unique-index/README.md +++ b/tests/vectors/unique-index/README.md @@ -19,7 +19,7 @@ reviewed commit. Other runtimes are held to the same canonical text and the same `" "` and `item:d` holds Sku `U+0085`. Written through a role Sku. - `blank-store.cc`: the same, only `item:c` and `item:d`. - `pair-store.cc`, `surrogate-store.cc`, `types-store.cc`, `repull-before.cc`, `repull-after.cc`, `alias-store.cc`, - `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`: single-file stores written like + `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`: single-file stores written like `duplicate-store.cc`, through roles, for the rows of `refusals.msgpack` that open or pull them. Schemas `tests.unique_item`, `tests.uq_pair` (Sku and Alt both unique), `tests.uq_alias` (Sku and `alt` both unique), `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus @@ -51,19 +51,20 @@ last succeeds and the last refuses with exactly the RULE text built from those f byte-identical. Schemas are those of the store files above; `second` is `Category` (plain) except in `tests.uq_pair` and `tests.uq_alias`, where it is `Alt` (unique, alias `Alt` and `alt` respectively). -The 13 rows of the spec are a floor; rows 14-22 extend it with orderings that a code-point, case-insensitive or +The 13 rows of the spec are a floor; rows 14-24 extend it with orderings that a code-point, case-insensitive or culture compare would flip. Rows, in file order: 1 batch-later-value-staged-first, 2 batch-surrogate-keys, 3 batch-interleaved-types, 4 batch-two-aliases, 5 held-then-landing, 6 held-in-the-least-group, 7 swap-off-a-held-value, 8 load-two-aliases, 9 load-surrogate-keys, 10 load-interleaved-types, 11 repull-held-pair, 12 load-repull-after, 13 commit-onto-a-loaded-holder, 14 batch-key-case (`B` before `a`), 15 batch-key-punctuation (`Z` before `_` before `a`), 16 batch-key-case-in-a-group, 17 batch-surrogate-groups (U+1F600 before U+E000, the reverse of code-point order), 18 batch-alias-case (alias `Sku` before `alt`), 19 load-alias-case, 20 load-key-case, 21 load-key-punctuation, 22 -load-surrogate-groups. Rows 19-22 open a store holding the records of rows 18, 14, 15 and 17, so a runtime with no batch +load-surrogate-groups, 23 batch-holders-by-case (`a` and `A` holding one value: `A` before `a`), 24 load-holders-by-case. +Rows 19-22 and 24 open a store holding the records of rows 18, 14, 15, 17 and 23, so a runtime with no batch commit still walks alias and cross-group key order. A runtime performs every row its API can perform and asserts the count it walked: -- C# and Rust: all 22. -- Python: 21, every row but 3 (`put_envelopes` takes one type). -- TypeScript: 11, rows 5, 8, 9, 10, 11, 12, 13, 19, 20, 21 and 22 (no batch commit, so no row 1-4, 6, 7 or 14-18; +- C# and Rust: all 24. +- Python: 23, every row but 3 (`put_envelopes` takes one type). +- TypeScript: 12, rows 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22 and 24 (no batch commit, so no row 1-4, 6, 7, 14-18 or 23; `variants:idea:unique-index-runtimes.ts-batch-commit`). diff --git a/tests/vectors/unique-index/holder-case-store.cc b/tests/vectors/unique-index/holder-case-store.cc new file mode 100644 index 0000000000000000000000000000000000000000..ed8f9a2ae3397342085ff4eacb7386ea87d2958d GIT binary patch literal 943 zcmcgqy>8nu5RQSoNVL`wC`zJ5ZAOP~9jc;BQ4}ZgXbX|#R-}p=MIItuQZhet zO;HbMT-bMLu)$iSIh3 Date: Wed, 11 Nov 2026 10:09:30 +0000 Subject: [PATCH 2/4] cultcache-rs: a unique-index refusal names the least violation by UTF-16 order; walk refusals.msgpack Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/src/lib.rs | 46 ++++-- packages/cultcache-rs/tests/unique_index.rs | 152 ++++++++++++++++---- 2 files changed, 158 insertions(+), 40 deletions(-) diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 11d21330..e5e88b9c 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -2738,6 +2738,16 @@ pub struct CultCache { } type IndexProjector = fn(&[u8]) -> Result>; +/// Orders by UTF-16 code unit (C# `StringComparer.Ordinal`), not by UTF-8 bytes or code point. +fn utf16_cmp(left: &str, right: &str) -> std::cmp::Ordering { + left.encode_utf16().cmp(right.encode_utf16()) +} + +/// Orders violations (alias, least key, second key, schema) field by field, each by UTF-16. +fn violation_cmp(left: &[&str; 4], right: &[&str; 4]) -> std::cmp::Ordering { + left.iter().zip(right).map(|(a, b)| utf16_cmp(a, b)).find(|order| order.is_ne()).unwrap_or(std::cmp::Ordering::Equal) +} + type IndexSlot = (String, String, String); type IndexHolders = BTreeMap; @@ -3424,22 +3434,36 @@ impl CultCache { groups[index].1.insert(key); } } - groups.sort_by(|left, right| left.0.1.cmp(&right.0.1)); - for (slot, mut holders) in groups { - if let Some(held_key) = held.get(&slot) + let mut least: Option<[&str; 4]> = None; + for (slot, holders) in &mut groups { + if let Some(held_key) = held.get(&*slot) && !last.contains_key(&(slot.0.as_str(), held_key.as_str())) { holders.insert(held_key.clone()); } - let mut ordered = holders.iter(); - if let (Some(first), Some(second)) = (ordered.next(), ordered.next()) { - return Err(anyhow!( - "Records {first} and {second} ({}) both hold unique index '{}'; an index value names one record.", - self.definitions.get(&slot.0).copied().unwrap_or(&slot.0), - slot.1 - )); + } + let mut ordered: Vec<(&IndexSlot, Vec<&String>)> = Vec::new(); + for (slot, holders) in &groups { + let mut keys: Vec<&String> = holders.iter().collect(); + keys.sort_by(|left, right| utf16_cmp(left, right)); + ordered.push((slot, keys)); + } + for (slot, keys) in &ordered { + if let [first, second, ..] = keys.as_slice() { + let schema = self.definitions.get(&slot.0).copied().unwrap_or(&slot.0); + let violation = [slot.1.as_str(), first.as_str(), second.as_str(), schema]; + if least.is_none_or(|best| violation_cmp(&violation, &best).is_lt()) { + least = Some(violation); + } } - let key = holders.into_iter().next().expect("a group holds at least one key"); + } + if let Some([alias, first, second, schema]) = least { + return Err(anyhow!( + "Records {first} and {second} ({schema}) both hold unique index '{alias}'; an index value names one record." + )); + } + for (slot, mut keys) in ordered.into_iter().map(|(slot, keys)| (slot.clone(), keys)) { + let key = keys.remove(0).clone(); change.taken.push((slot, key)); } Ok(change) diff --git a/packages/cultcache-rs/tests/unique_index.rs b/packages/cultcache-rs/tests/unique_index.rs index d0da4175..91ce8ca3 100644 --- a/packages/cultcache-rs/tests/unique_index.rs +++ b/packages/cultcache-rs/tests/unique_index.rs @@ -80,6 +80,41 @@ struct VBool { v: bool, } +/// The refusals vector's other schemas: `Sku` is unique in each; `Alt` is a second unique index in `uq_pair` (alias +/// `Alt`) and `uq_alias` (alias `alt`, sorting after `Sku` only under an ordinal compare); alpha and beta are plain. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_pair", schema = "tests.uq_pair")] +struct UqPair { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1, index = "Alt")] + alt: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_alias", schema = "tests.uq_alias")] +struct UqAlias { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1, index = "alt")] + alt: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_alpha", schema = "tests.uq_alpha")] +struct UqAlpha { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + second: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_beta", schema = "tests.uq_beta")] +struct UqBeta { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + second: String, +} + fn unique(sku: &str) -> Unique { Unique { sku: sku.into(), category: "c".into() } } @@ -339,35 +374,6 @@ fn a_batch_is_judged_whole() { assert_eq!(cache.get_by_index::("Sku", "D1").unwrap(), cache.get::("e").unwrap()); } -// The verdict and the named pair do not depend on key order, staging order, or alias declaration order. -#[test] -fn the_refusal_names_the_ordinal_pair_whatever_the_order() { - let orders: [[&str; 3]; 6] = - [["a", "b", "c"], ["a", "c", "b"], ["b", "a", "c"], ["b", "c", "a"], ["c", "a", "b"], ["c", "b", "a"]]; - for order in orders { - let mut cache = memory_cache(); - let rows: Vec<_> = order.iter().map(|key| (*key, unique("S"))).collect(); - let text = error_text(cache.put_prepared_batch(staged(&cache, &rows))); - assert_eq!(text, refusal("a", "b", "tests.unique_item", "Sku"), "{order:?}"); - assert!(skus(&cache).is_empty()); - } - // Both aliases collide in one batch: the alias sorting first is named, though `B` is declared first. - for order in [["x", "y"], ["y", "x"]] { - let mut cache = memory_cache(); - let rows: Vec<_> = order.iter().map(|key| (*key, multi(Some("s"), Some("t")))).collect(); - let text = error_text(cache.put_prepared_batch(staged(&cache, &rows))); - assert_eq!(text, refusal("x", "y", "Multi", "A")); - } - // A held key and a landing key: the pair is ordinal, not held-first. - let mut cache = memory_cache(); - cache.put("m", &unique("S")).unwrap(); - assert_eq!(error_text(cache.put("z", &unique("S"))), refusal("m", "z", "tests.unique_item", "Sku")); - assert_eq!(error_text(cache.put("a", &unique("S"))), refusal("a", "m", "tests.unique_item", "Sku")); - // A value that differs only by case or by a surrounding space is another value. - cache.put("n", &unique("s")).unwrap(); - cache.put("o", &unique("S ")).unwrap(); -} - #[test] fn a_load_of_a_duplicate_image_is_refused_and_keeps_the_prior_view() { let dir = tempfile::tempdir().unwrap(); @@ -561,6 +567,94 @@ fn vectors_stores_hold_their_verdicts() { } } +/// One op of a refusals row: `["open", file-or-nil]`, `["pull", file]` or `["commit", [[key, schema, sku, second], ...]]`. +#[derive(Deserialize)] +#[serde(untagged)] +enum Arg { + Records(Vec<(String, String, String, String)>), + File(Option), +} + +type RefusalRow = (String, Vec<(String, Arg)>, Option<(String, String, String, String)>); + +/// Rows `[label, ops, refusal]`: the pair a refusal names is a function of the state the admission would leave +/// (`tests/vectors/unique-index/README.md`, RULE), whatever the staging, file or pull order, key case or alias case. +#[test] +fn vectors_refusals_hold_the_rule() { + let bytes = std::fs::read(vectors().join("refusals.msgpack")).unwrap(); + let rows: Vec = rmp_serde::from_slice(&bytes).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut walked = 0; + for (index, (label, ops, refused)) in rows.into_iter().enumerate() { + let path = dir.path().join(format!("row-{index}.cc")); + let mut cache: Option = None; + let mut failure = None; + for (at, (kind, arg)) in ops.iter().enumerate() { + let last = at + 1 == ops.len(); + let mut before = path.exists().then(|| std::fs::read(&path).unwrap()); + let result = match (kind.as_str(), arg) { + ("open", Arg::File(file)) => { + if let Some(file) = file { + std::fs::copy(vectors().join(file), &path).unwrap(); + } + let mut opened = CultCache::new(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + let pulled = opened.pull_all_backing_stores(); + cache = Some(opened); + pulled + } + ("pull", Arg::File(Some(file))) => { + std::fs::copy(vectors().join(file), &path).unwrap(); + before = Some(std::fs::read(&path).unwrap()); + cache.as_mut().unwrap().pull_all_backing_stores() + } + ("commit", Arg::Records(records)) => { + let cache = cache.as_mut().unwrap(); + let staged: Vec<_> = records + .iter() + .map(|(key, schema, sku, second)| { + let (sku, second) = (sku.clone(), second.clone()); + match schema.as_str() { + "tests.unique_item" => cache.prepare_entry(key, &Unique { sku, category: second }).map(|(envelope, _)| envelope), + "tests.uq_pair" => cache.prepare_entry(key, &UqPair { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_alias" => cache.prepare_entry(key, &UqAlias { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_alpha" => cache.prepare_entry(key, &UqAlpha { sku, second }).map(|(envelope, _)| envelope), + "tests.uq_beta" => cache.prepare_entry(key, &UqBeta { sku, second }).map(|(envelope, _)| envelope), + other => panic!("{label}: unexpected schema {other}"), + } + .unwrap() + }) + .collect(); + cache.put_prepared_batch(staged) + } + other => panic!("{label}: unexpected op {:?}", other.0), + }; + if last { + failure = Some((result, before)); + } else { + result.unwrap_or_else(|error| panic!("{label}: op {at} should succeed: {error:#}")); + } + } + let (result, before) = failure.unwrap(); + match refused { + None => result.unwrap_or_else(|error| panic!("{label}: should succeed: {error:#}")), + Some((k1, k2, schema, alias)) => { + assert_eq!(error_text(result), refusal(&k1, &k2, &schema, &alias), "{label}"); + if let Some(before) = before { + assert_eq!(std::fs::read(&path).unwrap(), before, "{label}: the file is byte-identical"); + } + } + } + walked += 1; + } + assert_eq!(walked, 22); +} + // A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept. #[test] fn a_second_pull_replaces_the_holders_with_the_stored_image() { -- 2.47.3 From 677644c766e5cc1f6981dd7645f3c1ddd48c3304 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 10:10:01 +0000 Subject: [PATCH 3/4] cultcache-rs: walk the 24 refusals rows Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/unique_index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultcache-rs/tests/unique_index.rs b/packages/cultcache-rs/tests/unique_index.rs index 91ce8ca3..7307060a 100644 --- a/packages/cultcache-rs/tests/unique_index.rs +++ b/packages/cultcache-rs/tests/unique_index.rs @@ -652,7 +652,7 @@ fn vectors_refusals_hold_the_rule() { } walked += 1; } - assert_eq!(walked, 22); + assert_eq!(walked, 24); } // A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept. -- 2.47.3 From 3863cce1a15cb25eb09482cff09d87b78c417a26 Mon Sep 17 00:00:00 2001 From: Libby Date: Wed, 11 Nov 2026 11:45:31 +0000 Subject: [PATCH 4/4] cultcache-rs: walk the 33 refusals rows; name the row when a refusal is missing Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-rs/tests/unique_index.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cultcache-rs/tests/unique_index.rs b/packages/cultcache-rs/tests/unique_index.rs index 7307060a..1fc3c4eb 100644 --- a/packages/cultcache-rs/tests/unique_index.rs +++ b/packages/cultcache-rs/tests/unique_index.rs @@ -644,7 +644,8 @@ fn vectors_refusals_hold_the_rule() { match refused { None => result.unwrap_or_else(|error| panic!("{label}: should succeed: {error:#}")), Some((k1, k2, schema, alias)) => { - assert_eq!(error_text(result), refusal(&k1, &k2, &schema, &alias), "{label}"); + let error = result.err().unwrap_or_else(|| panic!("{label}: should refuse")); + assert_eq!(format!("{error:#}"), refusal(&k1, &k2, &schema, &alias), "{label}"); if let Some(before) = before { assert_eq!(std::fs::read(&path).unwrap(), before, "{label}: the file is byte-identical"); } @@ -652,7 +653,7 @@ fn vectors_refusals_hold_the_rule() { } walked += 1; } - assert_eq!(walked, 24); + assert_eq!(walked, 33); } // A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept. -- 2.47.3