diff --git a/packages/cultcache-py/tests/test_unique_index.py b/packages/cultcache-py/tests/test_unique_index.py index 9c19999d..accc27dd 100644 --- a/packages/cultcache-py/tests/test_unique_index.py +++ b/packages/cultcache-py/tests/test_unique_index.py @@ -475,12 +475,14 @@ class VectorTests(unittest.TestCase): "tests.unique_item": "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", "tests.uq_pair": "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", "tests.uq_alias": "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", + "tests.uq_wide": "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", "tests.uq_alpha": "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", "tests.uq_beta": "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", } aliases = { "tests.uq_pair": {"Sku": "sku", "Alt": "second"}, "tests.uq_alias": {"Sku": "sku", "alt": "second"}, + "tests.uq_wide": {"\uff21": "sku", "\U00010400": "second"}, # U+FF21 and U+10400: code-unit order puts the astral alias first } documents = { name: define_database_entry_type( @@ -535,7 +537,7 @@ class VectorTests(unittest.TestCase): self.assertEqual(str(caught.exception), refusal(k1, k2, schema, alias)) self.assertEqual(path.read_bytes() if path.exists() else b"", before) walked += 1 - self.assertEqual(walked, 31) + self.assertEqual(walked, 35) class ListStore(RecordingStore): diff --git a/packages/cultcache-rs/tests/unique_index.rs b/packages/cultcache-rs/tests/unique_index.rs index 1fc3c4eb..1132b2b5 100644 --- a/packages/cultcache-rs/tests/unique_index.rs +++ b/packages/cultcache-rs/tests/unique_index.rs @@ -98,6 +98,15 @@ struct UqAlias { #[cultcache(key = 1, index = "alt")] alt: String, } +/// Aliases U+FF21 and U+10400: UTF-16 puts the astral alias first (D801 DC00 < FF21), code-point order the other way. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_wide", schema = "tests.uq_wide")] +struct UqWide { + #[cultcache(key = 0, index = "\u{FF21}")] + sku: String, + #[cultcache(key = 1, index = "\u{10400}")] + alt: String, +} #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "uq_alpha", schema = "tests.uq_alpha")] struct UqAlpha { @@ -601,6 +610,7 @@ fn vectors_refusals_hold_the_rule() { opened.register_entry_type::().unwrap(); opened.register_entry_type::().unwrap(); opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); opened.register_entry_type::().unwrap(); opened.register_entry_type::().unwrap(); opened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); @@ -623,6 +633,7 @@ fn vectors_refusals_hold_the_rule() { "tests.unique_item" => cache.prepare_entry(key, &Unique { sku, category: second }).map(|(envelope, _)| envelope), "tests.uq_pair" => cache.prepare_entry(key, &UqPair { sku, alt: second }).map(|(envelope, _)| envelope), "tests.uq_alias" => cache.prepare_entry(key, &UqAlias { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_wide" => cache.prepare_entry(key, &UqWide { sku, alt: second }).map(|(envelope, _)| envelope), "tests.uq_alpha" => cache.prepare_entry(key, &UqAlpha { sku, second }).map(|(envelope, _)| envelope), "tests.uq_beta" => cache.prepare_entry(key, &UqBeta { sku, second }).map(|(envelope, _)| envelope), other => panic!("{label}: unexpected schema {other}"), @@ -653,7 +664,7 @@ fn vectors_refusals_hold_the_rule() { } walked += 1; } - assert_eq!(walked, 33); + assert_eq!(walked, 37); } // A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept. diff --git a/packages/cultcache-ts/test/unique-index.test.ts b/packages/cultcache-ts/test/unique-index.test.ts index ae296094..955c439a 100644 --- a/packages/cultcache-ts/test/unique-index.test.ts +++ b/packages/cultcache-ts/test/unique-index.test.ts @@ -359,11 +359,12 @@ test("vectors duplicate-store.cc and blank-store.cc hold their verdicts", async // refusals.msgpack: rows [label, ops, refusal] written by the C# reference. TypeScript has no batch commit // (variants:idea:unique-index-runtimes.ts-batch-commit), so it performs every row whose commits each hold one record: -// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32 and 33 of the README (18). +// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 of the README (20). const REFUSAL_SCHEMAS: [string, string, string, Record][] = [ ["unique_item", "tests.unique_item", "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", { Sku: "sku" }], ["uq_pair", "tests.uq_pair", "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", { Sku: "sku", Alt: "second" }], ["uq_alias", "tests.uq_alias", "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", { Sku: "sku", alt: "second" }], + ["uq_wide", "tests.uq_wide", "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", { "\uFF21": "sku", "\u{10400}": "second" }], ["uq_alpha", "tests.uq_alpha", "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", { Sku: "sku" }], ["uq_beta", "tests.uq_beta", "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", { Sku: "sku" }], ]; @@ -424,7 +425,7 @@ test("vectors refusals.msgpack: every row TS can perform", async () => { } finally { await rm(dir, { recursive: true, force: true }); } - assert.equal(walked, 18); + assert.equal(walked, 20); }); test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => { diff --git a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs index 23520310..1fd026ae 100644 --- a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs @@ -316,6 +316,12 @@ namespace GameCult.Caching.Tests EmittedDocumentTypes.Emit("UqVecAliasIndex", "tests.uq_alias", "tests.uq_alias.v1", new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "alt") }), "Alt"), + new("tests.uq_wide", + EmittedDocumentTypes.Emit("UqVecWideRole", "tests.uq_wide", "tests.uq_wide.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "\uFF21"), new("Alt", typeof(string), 1, Role: "\U00010400") }), + EmittedDocumentTypes.Emit("UqVecWideIndex", "tests.uq_wide", "tests.uq_wide.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "\uFF21"), new("Alt", typeof(string), 1, Index: "\U00010400") }), + "Alt"), Plain("Alpha", "tests.uq_alpha"), Plain("Beta", "tests.uq_beta"), }; @@ -333,6 +339,7 @@ namespace GameCult.Caching.Tests private const string Item = "tests.unique_item"; private const string Pair = "tests.uq_pair"; private const string Alias = "tests.uq_alias"; + private const string Wide = "tests.uq_wide"; private const string Alpha = "tests.uq_alpha"; private const string Beta = "tests.uq_beta"; @@ -357,6 +364,10 @@ namespace GameCult.Caching.Tests private static readonly Rec[] DistinctValueRecords = Items_( ("k:1", "S"), ("k:2", "s"), ("k:3", "S "), ("k:4", "\u00E9"), ("k:5", "e\u0301"), ("k:6", "\uFF33"), ("k:7", " S")); private static readonly Rec[] NormalKeyRecords = Items_(("\u00E9", "S"), ("e\u0301", "S")); + // Three holders of one value: code-unit order is e+U+0302, f, U+00E9; NFC would order f, U+00E9, e+U+0302 and NFD U+00E9, e+U+0302, f. + private static readonly Rec[] ThreeHolderRecords = Items_(("e\u0302", "S"), ("f", "S"), ("\u00E9", "S")); + // Both records violate both aliases: by code unit U+10400 (D801 DC00) precedes U+FF21, by code point U+FF21 does. + private static readonly Rec[] WideRecords = { new("item:a", Wide, "S", "X"), new("item:b", Wide, "S", "X") }; private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T")); private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T")); @@ -368,7 +379,7 @@ namespace GameCult.Caching.Tests ("surrogate-groups-store.cc", SurrogateGroupRecords), ("holder-case-store.cc", HolderCaseRecords), ("cross-schema-store.cc", CrossSchemaRecords), ("distinct-values-store.cc", DistinctValueRecords), - ("normal-keys-store.cc", NormalKeyRecords), + ("normal-keys-store.cc", NormalKeyRecords), ("normal-three-store.cc", ThreeHolderRecords), ("wide-store.cc", WideRecords), }; private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku"); @@ -379,6 +390,8 @@ namespace GameCult.Caching.Tests private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku"); private static readonly Refusal HolderCaseRefusal = new("A", "a", Item, "Sku"); private static readonly Refusal NormalKeyRefusal = new("e\u0301", "\u00E9", Item, "Sku"); + private static readonly Refusal ThreeHolderRefusal = new("e\u0302", "f", Item, "Sku"); + private static readonly Refusal WideRefusal = new("item:a", "item:b", Wide, "\U00010400"); private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku"); private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku"); @@ -430,6 +443,12 @@ namespace GameCult.Caching.Tests new Refusal("a", "m", Pair, "Alt")), new("put-onto-a-held-alias-pair", new[] { Open(), Commit(new Rec("m", Alias, "S", "X")), Commit(new Rec("a", Alias, "S", "X")) }, new Refusal("a", "m", Alias, "Sku")), + // Rows 34-35: three holders of one value; the two least by code unit are named, which no NFC, NFD or culture sort of the keys yields. + new("batch-three-holders-normal-form", new[] { Open(), Commit(ThreeHolderRecords) }, ThreeHolderRefusal), + new("load-three-holders-normal-form", new[] { Open("normal-three-store.cc") }, ThreeHolderRefusal), + // Rows 36-37: two records violating two aliases, U+FF21 and U+10400; aliases compare by code unit, so U+10400 is named. + new("batch-alias-code-unit", new[] { Open(), Commit(WideRecords) }, WideRefusal), + new("load-alias-code-unit", new[] { Open("wide-store.cc") }, WideRefusal), }; private static byte[] RefusalStoreBytes(Rec[] records) @@ -623,7 +642,7 @@ namespace GameCult.Caching.Tests } Assert.That(failed, Is.Empty, "rows that failed"); - Assert.That(walked, Is.EqualTo(33)); + Assert.That(walked, Is.EqualTo(37)); } } } diff --git a/tests/vectors/unique-index/README.md b/tests/vectors/unique-index/README.md index 45e6ef71..55b393f3 100644 --- a/tests/vectors/unique-index/README.md +++ b/tests/vectors/unique-index/README.md @@ -19,14 +19,15 @@ reviewed commit. Other runtimes are held to the same canonical text and the same `" "` and `item:d` holds Sku `U+0085`. Written through a role Sku. - `blank-store.cc`: the same, only `item:c` and `item:d`. - `pair-store.cc`, `surrogate-store.cc`, `types-store.cc`, `repull-before.cc`, `repull-after.cc`, `alias-store.cc`, - `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`, `cross-schema-store.cc`, `distinct-values-store.cc`, `normal-keys-store.cc`: single-file stores written like + `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`, `cross-schema-store.cc`, `distinct-values-store.cc`, `normal-keys-store.cc`, `normal-three-store.cc`, `wide-store.cc`: single-file stores written like `duplicate-store.cc`, through roles, for the rows of `refusals.msgpack` that open or pull them. Schemas `tests.unique_item`, `tests.uq_pair` (Sku and Alt both unique), `tests.uq_alias` (Sku and `alt` both unique), - `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus + `tests.uq_wide` (Sku and Alt both unique, aliases U+FF21 and U+10400), `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus `.v1`. The schema ids C# writes in the store catalogs and on every record: - `tests.unique_item`: `sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882` - `tests.uq_pair`: `sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60` - `tests.uq_alias`: `sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f` + - `tests.uq_wide`: `sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f` - `tests.uq_alpha`: `sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389` - `tests.uq_beta`: `sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586` - `outcomes.msgpack`: rows `[file, unique alias or nil, open|refused, k1 or nil, k2 or nil]`. Opening `file` with that @@ -51,7 +52,7 @@ last succeeds and the last refuses with exactly the RULE text built from those f byte-identical. Schemas are those of the store files above; `second` is `Category` (plain) except in `tests.uq_pair` and `tests.uq_alias`, where it is `Alt` (unique, alias `Alt` and `alt` respectively). -The 13 rows of the spec are a floor; rows 14-33 extend it with orderings that a code-point, case-insensitive or +The 13 rows of the spec are a floor; rows 14-37 extend it with orderings that a code-point, case-insensitive or culture compare would flip. Rows, in file order: 1 batch-later-value-staged-first, 2 batch-surrogate-keys, 3 batch-interleaved-types, 4 batch-two-aliases, 5 held-then-landing, 6 held-in-the-least-group, 7 swap-off-a-held-value, 8 load-two-aliases, 9 load-surrogate-keys, 10 load-interleaved-types, 11 repull-held-pair, 12 load-repull-after, 13 @@ -63,17 +64,20 @@ load-surrogate-groups, 23 batch-holders-by-case (`a` and `A` holding one value: load-value-in-two-schemas, 28 batch-distinct-values (`S`, `s`, `S `, ` S`, U+00E9, `e`+U+0301, U+FF33 are seven values: success), 29 load-distinct-values, 30 batch-keys-by-normal-form (keys U+00E9 and `e`+U+0301 holding one value are two holders, `e`+U+0301 first), 31 load-keys-by-normal-form, 32 put-onto-a-held-pair (held `m` and landing `a` share Sku and Alt in `tests.uq_pair`, one record per -commit: `Alt` is named before `Sku`), 33 put-onto-a-held-alias-pair (the same in `tests.uq_alias`: `Sku` before `alt`). -Rows 19-22, 24, 27, 29 and 31 open a store holding the records of rows 18, 14, 15, 17, 23, 26, 28 and 30, so a runtime with no batch +commit: `Alt` is named before `Sku`), 33 put-onto-a-held-alias-pair (the same in `tests.uq_alias`: `Sku` before `alt`), 34 batch-three-holders-normal-form +(keys `e`+U+0302, `f` and U+00E9 holding one value: the two least by code unit, `e`+U+0302 and `f`, are named; an NFC, NFD or culture sort of the keys names another pair), 35 +load-three-holders-normal-form, 36 batch-alias-code-unit (two records each holding the same value under both aliases of `tests.uq_wide`, U+FF21 and U+10400: by UTF-16 code +unit U+10400 (D801 DC00) precedes U+FF21, by code point it follows it, so the refusal names alias U+10400), 37 load-alias-code-unit. +Rows 19-22, 24, 27, 29, 31, 35 and 37 open a store holding the records of rows 18, 14, 15, 17, 23, 26, 28, 30, 34 and 36, so a runtime with no batch commit still walks alias and cross-group key order. A runtime performs every row its API can perform and asserts the count it walks. These are the counts a runtime must assert, not a statement of which runtimes walk them today: -- C# and Rust: all 33. -- Python: 31, every row but 3 and 26 (`put_envelopes` takes one type, so no batch of two types). -- TypeScript: 18, the rows with no multi-record commit: 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27, 29, 31, 32 and 33 - (no batch commit, so no row 1-4, 6, 7, 14-18, 23, 26, 28 or 30; `variants:idea:unique-index-runtimes.ts-batch-commit`). +- C# and Rust: all 37. +- Python: 35, every row but 3 and 26 (`put_envelopes` takes one type, so no batch of two types). +- TypeScript: 20, the rows with no multi-record commit: 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 + (no batch commit, so no row 1-4, 6, 7, 14-18, 23, 26, 28, 30, 34 or 36; `variants:idea:unique-index-runtimes.ts-batch-commit`). Not pinned: the schema-name tiebreak. Record keys are unique across a whole cache, so two types never hold the same key and two groups can never tie on alias and key pair; the final `schema` comparison cannot decide a refusal. diff --git a/tests/vectors/unique-index/normal-three-store.cc b/tests/vectors/unique-index/normal-three-store.cc new file mode 100644 index 00000000..580ae038 Binary files /dev/null and b/tests/vectors/unique-index/normal-three-store.cc differ diff --git a/tests/vectors/unique-index/refusals.msgpack b/tests/vectors/unique-index/refusals.msgpack index 34fc402d..01dcc9e1 100644 Binary files a/tests/vectors/unique-index/refusals.msgpack and b/tests/vectors/unique-index/refusals.msgpack differ diff --git a/tests/vectors/unique-index/wide-store.cc b/tests/vectors/unique-index/wide-store.cc new file mode 100644 index 00000000..2dcc1e4a Binary files /dev/null and b/tests/vectors/unique-index/wide-store.cc differ