packages/cultcache-py/tests/test_unique_index.py

@@ -475,12 +475,14 @@ class VectorTests(unittest.TestCase):
             "tests.unique_item": "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882",
             "tests.uq_pair": "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60",
             "tests.uq_alias": "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f",
+            "tests.uq_wide": "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f",
             "tests.uq_alpha": "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389",
             "tests.uq_beta": "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586",
         }
         aliases = {
             "tests.uq_pair": {"Sku": "sku", "Alt": "second"},
             "tests.uq_alias": {"Sku": "sku", "alt": "second"},
+            "tests.uq_wide": {"\uff21": "sku", "\U00010400": "second"},  # U+FF21 and U+10400: code-unit order puts the astral alias first
         }
         documents = {
             name: define_database_entry_type(
@@ -535,7 +537,7 @@ class VectorTests(unittest.TestCase):
                     self.assertEqual(str(caught.exception), refusal(k1, k2, schema, alias))
                     self.assertEqual(path.read_bytes() if path.exists() else b"", before)
                 walked += 1
-        self.assertEqual(walked, 31)
+        self.assertEqual(walked, 35)
 
 
 class ListStore(RecordingStore):

packages/cultcache-rs/tests/unique_index.rs

@@ -98,6 +98,15 @@ struct UqAlias {
     #[cultcache(key = 1, index = "alt")]
     alt: String,
 }
+/// Aliases U+FF21 and U+10400: UTF-16 puts the astral alias first (D801 DC00 < FF21), code-point order the other way.
+#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
+#[cultcache(type = "uq_wide", schema = "tests.uq_wide")]
+struct UqWide {
+    #[cultcache(key = 0, index = "\u{FF21}")]
+    sku: String,
+    #[cultcache(key = 1, index = "\u{10400}")]
+    alt: String,
+}
 #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
 #[cultcache(type = "uq_alpha", schema = "tests.uq_alpha")]
 struct UqAlpha {
@@ -601,6 +610,7 @@ fn vectors_refusals_hold_the_rule() {
                     opened.register_entry_type::<Unique>().unwrap();
                     opened.register_entry_type::<UqPair>().unwrap();
                     opened.register_entry_type::<UqAlias>().unwrap();
+                    opened.register_entry_type::<UqWide>().unwrap();
                     opened.register_entry_type::<UqAlpha>().unwrap();
                     opened.register_entry_type::<UqBeta>().unwrap();
                     opened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
@@ -623,6 +633,7 @@ fn vectors_refusals_hold_the_rule() {
                                 "tests.unique_item" => cache.prepare_entry(key, &Unique { sku, category: second }).map(|(envelope, _)| envelope),
                                 "tests.uq_pair" => cache.prepare_entry(key, &UqPair { sku, alt: second }).map(|(envelope, _)| envelope),
                                 "tests.uq_alias" => cache.prepare_entry(key, &UqAlias { sku, alt: second }).map(|(envelope, _)| envelope),
+                                "tests.uq_wide" => cache.prepare_entry(key, &UqWide { sku, alt: second }).map(|(envelope, _)| envelope),
                                 "tests.uq_alpha" => cache.prepare_entry(key, &UqAlpha { sku, second }).map(|(envelope, _)| envelope),
                                 "tests.uq_beta" => cache.prepare_entry(key, &UqBeta { sku, second }).map(|(envelope, _)| envelope),
                                 other => panic!("{label}: unexpected schema {other}"),
@@ -653,7 +664,7 @@ fn vectors_refusals_hold_the_rule() {
         }
         walked += 1;
     }
-    assert_eq!(walked, 33);
+    assert_eq!(walked, 37);
 }
 
 // A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept.

packages/cultcache-ts/test/unique-index.test.ts

@@ -359,11 +359,12 @@ test("vectors duplicate-store.cc and blank-store.cc hold their verdicts", async
 
 // refusals.msgpack: rows [label, ops, refusal] written by the C# reference. TypeScript has no batch commit
 // (variants:idea:unique-index-runtimes.ts-batch-commit), so it performs every row whose commits each hold one record:
-// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32 and 33 of the README (18).
+// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 of the README (20).
 const REFUSAL_SCHEMAS: [string, string, string, Record<string, "sku" | "second">][] = [
   ["unique_item", "tests.unique_item", "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", { Sku: "sku" }],
   ["uq_pair", "tests.uq_pair", "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", { Sku: "sku", Alt: "second" }],
   ["uq_alias", "tests.uq_alias", "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", { Sku: "sku", alt: "second" }],
+  ["uq_wide", "tests.uq_wide", "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", { "\uFF21": "sku", "\u{10400}": "second" }],
   ["uq_alpha", "tests.uq_alpha", "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", { Sku: "sku" }],
   ["uq_beta", "tests.uq_beta", "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", { Sku: "sku" }],
 ];
@@ -424,7 +425,7 @@ test("vectors refusals.msgpack: every row TS can perform", async () => {
   } finally {
     await rm(dir, { recursive: true, force: true });
   }
-  assert.equal(walked, 18);
+  assert.equal(walked, 20);
 });
 
 test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => {

tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs

@@ -316,6 +316,12 @@ namespace GameCult.Caching.Tests
                 EmittedDocumentTypes.Emit("UqVecAliasIndex", "tests.uq_alias", "tests.uq_alias.v1",
                     new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "alt") }),
                 "Alt"),
+            new("tests.uq_wide",
+                EmittedDocumentTypes.Emit("UqVecWideRole", "tests.uq_wide", "tests.uq_wide.v1",
+                    new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "\uFF21"), new("Alt", typeof(string), 1, Role: "\U00010400") }),
+                EmittedDocumentTypes.Emit("UqVecWideIndex", "tests.uq_wide", "tests.uq_wide.v1",
+                    new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "\uFF21"), new("Alt", typeof(string), 1, Index: "\U00010400") }),
+                "Alt"),
             Plain("Alpha", "tests.uq_alpha"),
             Plain("Beta", "tests.uq_beta"),
         };
@@ -333,6 +339,7 @@ namespace GameCult.Caching.Tests
         private const string Item = "tests.unique_item";
         private const string Pair = "tests.uq_pair";
         private const string Alias = "tests.uq_alias";
+        private const string Wide = "tests.uq_wide";
         private const string Alpha = "tests.uq_alpha";
         private const string Beta = "tests.uq_beta";
 
@@ -357,6 +364,10 @@ namespace GameCult.Caching.Tests
         private static readonly Rec[] DistinctValueRecords = Items_(
             ("k:1", "S"), ("k:2", "s"), ("k:3", "S "), ("k:4", "\u00E9"), ("k:5", "e\u0301"), ("k:6", "\uFF33"), ("k:7", " S"));
         private static readonly Rec[] NormalKeyRecords = Items_(("\u00E9", "S"), ("e\u0301", "S"));
+        // Three holders of one value: code-unit order is e+U+0302, f, U+00E9; NFC would order f, U+00E9, e+U+0302 and NFD U+00E9, e+U+0302, f.
+        private static readonly Rec[] ThreeHolderRecords = Items_(("e\u0302", "S"), ("f", "S"), ("\u00E9", "S"));
+        // Both records violate both aliases: by code unit U+10400 (D801 DC00) precedes U+FF21, by code point U+FF21 does.
+        private static readonly Rec[] WideRecords = { new("item:a", Wide, "S", "X"), new("item:b", Wide, "S", "X") };
         private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T"));
         private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T"));
 
@@ -368,7 +379,7 @@ namespace GameCult.Caching.Tests
             ("surrogate-groups-store.cc", SurrogateGroupRecords),
             ("holder-case-store.cc", HolderCaseRecords),
             ("cross-schema-store.cc", CrossSchemaRecords), ("distinct-values-store.cc", DistinctValueRecords),
-            ("normal-keys-store.cc", NormalKeyRecords),
+            ("normal-keys-store.cc", NormalKeyRecords), ("normal-three-store.cc", ThreeHolderRecords), ("wide-store.cc", WideRecords),
         };
 
         private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku");
@@ -379,6 +390,8 @@ namespace GameCult.Caching.Tests
         private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku");
         private static readonly Refusal HolderCaseRefusal = new("A", "a", Item, "Sku");
         private static readonly Refusal NormalKeyRefusal = new("e\u0301", "\u00E9", Item, "Sku");
+        private static readonly Refusal ThreeHolderRefusal = new("e\u0302", "f", Item, "Sku");
+        private static readonly Refusal WideRefusal = new("item:a", "item:b", Wide, "\U00010400");
         private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku");
         private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku");
 
@@ -430,6 +443,12 @@ namespace GameCult.Caching.Tests
                 new Refusal("a", "m", Pair, "Alt")),
             new("put-onto-a-held-alias-pair", new[] { Open(), Commit(new Rec("m", Alias, "S", "X")), Commit(new Rec("a", Alias, "S", "X")) },
                 new Refusal("a", "m", Alias, "Sku")),
+            // Rows 34-35: three holders of one value; the two least by code unit are named, which no NFC, NFD or culture sort of the keys yields.
+            new("batch-three-holders-normal-form", new[] { Open(), Commit(ThreeHolderRecords) }, ThreeHolderRefusal),
+            new("load-three-holders-normal-form", new[] { Open("normal-three-store.cc") }, ThreeHolderRefusal),
+            // Rows 36-37: two records violating two aliases, U+FF21 and U+10400; aliases compare by code unit, so U+10400 is named.
+            new("batch-alias-code-unit", new[] { Open(), Commit(WideRecords) }, WideRefusal),
+            new("load-alias-code-unit", new[] { Open("wide-store.cc") }, WideRefusal),
         };
 
         private static byte[] RefusalStoreBytes(Rec[] records)
@@ -623,7 +642,7 @@ namespace GameCult.Caching.Tests
             }
 
             Assert.That(failed, Is.Empty, "rows that failed");
-            Assert.That(walked, Is.EqualTo(33));
+            Assert.That(walked, Is.EqualTo(37));
         }
     }
 }

tests/vectors/unique-index/README.md

@@ -19,14 +19,15 @@ reviewed commit. Other runtimes are held to the same canonical text and the same
   `" "` and `item:d` holds Sku `U+0085`. Written through a role Sku.
 - `blank-store.cc`: the same, only `item:c` and `item:d`.
 - `pair-store.cc`, `surrogate-store.cc`, `types-store.cc`, `repull-before.cc`, `repull-after.cc`, `alias-store.cc`,
-  `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`, `cross-schema-store.cc`, `distinct-values-store.cc`, `normal-keys-store.cc`: single-file stores written like
+  `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`, `cross-schema-store.cc`, `distinct-values-store.cc`, `normal-keys-store.cc`, `normal-three-store.cc`, `wide-store.cc`: single-file stores written like
   `duplicate-store.cc`, through roles, for the rows of `refusals.msgpack` that open or pull them. Schemas
   `tests.unique_item`, `tests.uq_pair` (Sku and Alt both unique), `tests.uq_alias` (Sku and `alt` both unique),
-  `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus
+  `tests.uq_wide` (Sku and Alt both unique, aliases U+FF21 and U+10400), `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus
   `.v1`. The schema ids C# writes in the store catalogs and on every record:
   - `tests.unique_item`: `sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882`
   - `tests.uq_pair`: `sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60`
   - `tests.uq_alias`: `sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f`
+  - `tests.uq_wide`: `sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f`
   - `tests.uq_alpha`: `sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389`
   - `tests.uq_beta`: `sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586`
 - `outcomes.msgpack`: rows `[file, unique alias or nil, open|refused, k1 or nil, k2 or nil]`. Opening `file` with that
@@ -51,7 +52,7 @@ last succeeds and the last refuses with exactly the RULE text built from those f
 byte-identical. Schemas are those of the store files above; `second` is `Category` (plain) except in `tests.uq_pair`
 and `tests.uq_alias`, where it is `Alt` (unique, alias `Alt` and `alt` respectively).
 
-The 13 rows of the spec are a floor; rows 14-33 extend it with orderings that a code-point, case-insensitive or
+The 13 rows of the spec are a floor; rows 14-37 extend it with orderings that a code-point, case-insensitive or
 culture compare would flip. Rows, in file order: 1 batch-later-value-staged-first, 2 batch-surrogate-keys, 3
 batch-interleaved-types, 4 batch-two-aliases, 5 held-then-landing, 6 held-in-the-least-group, 7 swap-off-a-held-value,
 8 load-two-aliases, 9 load-surrogate-keys, 10 load-interleaved-types, 11 repull-held-pair, 12 load-repull-after, 13
@@ -63,17 +64,20 @@ load-surrogate-groups, 23 batch-holders-by-case (`a` and `A` holding one value:
 load-value-in-two-schemas, 28 batch-distinct-values (`S`, `s`, `S `, ` S`, U+00E9, `e`+U+0301, U+FF33 are seven values: success), 29
 load-distinct-values, 30 batch-keys-by-normal-form (keys U+00E9 and `e`+U+0301 holding one value are two holders, `e`+U+0301 first),
 31 load-keys-by-normal-form, 32 put-onto-a-held-pair (held `m` and landing `a` share Sku and Alt in `tests.uq_pair`, one record per
-commit: `Alt` is named before `Sku`), 33 put-onto-a-held-alias-pair (the same in `tests.uq_alias`: `Sku` before `alt`).
-Rows 19-22, 24, 27, 29 and 31 open a store holding the records of rows 18, 14, 15, 17, 23, 26, 28 and 30, so a runtime with no batch
+commit: `Alt` is named before `Sku`), 33 put-onto-a-held-alias-pair (the same in `tests.uq_alias`: `Sku` before `alt`), 34 batch-three-holders-normal-form
+(keys `e`+U+0302, `f` and U+00E9 holding one value: the two least by code unit, `e`+U+0302 and `f`, are named; an NFC, NFD or culture sort of the keys names another pair), 35
+load-three-holders-normal-form, 36 batch-alias-code-unit (two records each holding the same value under both aliases of `tests.uq_wide`, U+FF21 and U+10400: by UTF-16 code
+unit U+10400 (D801 DC00) precedes U+FF21, by code point it follows it, so the refusal names alias U+10400), 37 load-alias-code-unit.
+Rows 19-22, 24, 27, 29, 31, 35 and 37 open a store holding the records of rows 18, 14, 15, 17, 23, 26, 28, 30, 34 and 36, so a runtime with no batch
 commit still walks alias and cross-group key order.
 
 A runtime performs every row its API can perform and asserts the count it walks. These are the counts a runtime must
 assert, not a statement of which runtimes walk them today:
 
-- C# and Rust: all 33.
-- Python: 31, every row but 3 and 26 (`put_envelopes` takes one type, so no batch of two types).
-- TypeScript: 18, the rows with no multi-record commit: 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27, 29, 31, 32 and 33
-  (no batch commit, so no row 1-4, 6, 7, 14-18, 23, 26, 28 or 30; `variants:idea:unique-index-runtimes.ts-batch-commit`).
+- C# and Rust: all 37.
+- Python: 35, every row but 3 and 26 (`put_envelopes` takes one type, so no batch of two types).
+- TypeScript: 20, the rows with no multi-record commit: 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27, 29, 31, 32, 33, 35 and 37
+  (no batch commit, so no row 1-4, 6, 7, 14-18, 23, 26, 28, 30, 34 or 36; `variants:idea:unique-index-runtimes.ts-batch-commit`).
 
 Not pinned: the schema-name tiebreak. Record keys are unique across a whole cache, so two types never hold the same key and
 two groups can never tie on alias and key pair; the final `schema` comparison cannot decide a refusal.