diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index 1a838e94..959171ce 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -1,11 +1,18 @@ name: CultMesh portability and performance +# The forge runs one job at a time and this workflow holds the slot 14 to 30 minutes, so it runs once per change: +# on the pull request (never again on the push that merges it; a break that merges from a stale base waits for the +# schedule), skipped for design-note-only changes, and on a +# daily schedule so main is still exercised when nobody opens a pull request. on: - push: - branches: - - main - - "codex/**" pull_request: + paths-ignore: + # Not `**/*.md` or `docs/**`: docs/start/README.md is read by the first-call tests of every runtime. + - "docs/*.md" + - "research/**" + - ".voidbot/**" + schedule: + - cron: "17 3 * * *" workflow_dispatch: concurrency: diff --git a/.github/workflows/quic-native.yml b/.github/workflows/quic-native.yml index 6a4c9cdf..8f1fed18 100644 --- a/.github/workflows/quic-native.yml +++ b/.github/workflows/quic-native.yml @@ -9,13 +9,21 @@ name: CultMesh QUIC native # from the pinned debian:13 container below, never from a local or Yggdrasil # build. +# Only a change to the committed binaries, their manifest or this workflow can change what verify-committed checks, +# so a branch push or pull request runs it for those paths alone; a tag always runs it. on: push: branches: - main + paths: + - "packages/cultmesh-ts/native/**" + - ".github/workflows/quic-native.yml" tags: - "cultmesh-quic-native-v*" pull_request: + paths: + - "packages/cultmesh-ts/native/**" + - ".github/workflows/quic-native.yml" workflow_dispatch: concurrency: diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index 1ddc80e8..8d35fbb6 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -146,6 +146,9 @@ A malformed put or delete is `message_invalid`, not a per-operation code: the pe commit's changes as one commit, so the replica judges the result (a swap of two unique-index values) and not the state between records. A replica whose cache refuses a commit is faulted, never silent. `commitId` is opaque and valid only within the subscription. The C# subscription server and database server tag commits; other runtimes adopt next. +- The C# writer sets every key of a `database_change_raw` message, nil where unset: a removed change carries `document` + nil, an added or updated one `recordKey` and `schemaId` nil. The schema accepts those nils; an added or updated change + still needs a document, and a removed one a `recordKey` and `schemaId`. - Client-owned input documents may be predicted locally only inside explicit `CultNetClientAuthorityScope` declarations. - Predicted input state is not authoritative shared state. The shard log still diff --git a/contracts/cultnet/cultnet.database-change-raw.schema.json b/contracts/cultnet/cultnet.database-change-raw.schema.json index f1cd97eb..badada4a 100644 --- a/contracts/cultnet/cultnet.database-change-raw.schema.json +++ b/contracts/cultnet/cultnet.database-change-raw.schema.json @@ -10,17 +10,29 @@ "messageId": { "type": "string", "minLength": 1 }, "subscriptionId": { "type": "string", "minLength": 1 }, "changeKind": { "enum": ["added", "updated", "removed"] }, - "document": { "$ref": "./cultnet.raw-document-record.schema.json" }, - "recordKey": { "type": "string", "minLength": 1 }, - "schemaId": { "type": "string", "minLength": 1 }, + "document": { + "oneOf": [{ "type": "null" }, { "$ref": "./cultnet.raw-document-record.schema.json" }] + }, + "recordKey": { "type": ["string", "null"], "minLength": 1 }, + "schemaId": { "type": ["string", "null"], "minLength": 1 }, "commitId": { "type": ["string", "null"], "minLength": 1 }, "commitCount": { "type": ["integer", "null"], "minimum": 2 } }, - "allOf": [ + "oneOf": [ { - "if": { "properties": { "changeKind": { "const": "removed" } } }, - "then": { "required": ["recordKey", "schemaId"] }, - "else": { "required": ["document"] } + "properties": { + "changeKind": { "const": "removed" }, + "recordKey": { "type": "string" }, + "schemaId": { "type": "string" } + }, + "required": ["recordKey", "schemaId"] + }, + { + "properties": { + "changeKind": { "enum": ["added", "updated"] }, + "document": { "$ref": "./cultnet.raw-document-record.schema.json" } + }, + "required": ["document"] } ] } diff --git a/docs/document-variants-cut.md b/docs/document-variants-cut.md index edf86d6a..606e46b0 100644 --- a/docs/document-variants-cut.md +++ b/docs/document-variants-cut.md @@ -701,6 +701,7 @@ checked by the operator in the editor. - Run `MintElementIds` once over `GameData/Aetheria.cc`. - AetherDb migrations skip variants or use the variant API (Q1a). - Re-author the families listed in section 6. +- Hull families live in each ship's own `.cc`, not in `GameData/Aetheria.cc`, because hulls are mod-owned packages (Nibu's ruling about her data, 2026-11-11, #minds). ### C5. CultNet deltas (C# reference) diff --git a/docs/document-variants-id-sweep.md b/docs/document-variants-id-sweep.md index 03fad9cb..74ecd023 100644 --- a/docs/document-variants-id-sweep.md +++ b/docs/document-variants-id-sweep.md @@ -4,6 +4,8 @@ Data for the per-consumer sweep in docs/document-variants-cut.md, C2. Every obje Method: a static scan of `F:\Projects\` C# files for types marked `[CultDocument(`, walked through keyed members, base classes, dictionary values and `[Union]` subtypes. Read-only; no consumer was edited. Worktree copies (`Aetheria-catalog`, `CultLib-*`, and so on) are excluded. Struct elements are values and need no id. Non-C# consumers carry the id as an ordinary member and are not listed. The scan is text-based: a document registered without a literal `[CultDocument(` attribute, or a type whose members lack an integer `[Key]`, is missed, so the sweep pass should confirm each repo by registering its types once. +A type that already has a readable string id keeps it and adds a trailing keyed string member marked `[CultElementId(nameof(ExistingId))]`; see the Element ids section of `src/GameCult.Caching/README.md`. + Lines: `Type file:line (via Owner.Member)`, paths relative to the consumer root. diff --git a/docs/nuget-packaging.md b/docs/nuget-packaging.md index 301e54eb..7894fe5e 100644 --- a/docs/nuget-packaging.md +++ b/docs/nuget-packaging.md @@ -25,5 +25,13 @@ closure through UPM rather than NuGet restore. Install the Unity package from the immutable release tag: ```text -https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v2.0.0 +https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v3.0.0 ``` + +## Rolling back across the store format + +A consumer that upgrades and writes a store holding an element id leaves a +`cultcache.store.v3` single-file store (`v5` directory store). Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. There is +no general carry-back; restore from a copy taken before upgrading, and upgrade +readers before writers. Detail: the Store format and rollback section of +`src/GameCult.Caching/README.md`. diff --git a/docs/runtime-parity-scope.md b/docs/runtime-parity-scope.md index e001bb1d..2f54e2fe 100644 --- a/docs/runtime-parity-scope.md +++ b/docs/runtime-parity-scope.md @@ -16,7 +16,7 @@ Cross-runtime parity has three layers: Every store reader refuses, by name, a header or record slot it does not understand (vectors: `tests/vectors/document-variants-c0`). Element ids: a store that can hold them declares `cultcache.store.v3`; TypeScript, - Python and Rust read it and keep the header, and readers older than ids refuse it + Python and Rust read it and keep the header, and the C# runtime decides the mark from content, and these runtimes never drop it. A C# reader from `cultlib-unity-v2.0.0` on refuses a v3 single-file store it cannot read (earlier pins do not) (vectors: `tests/vectors/document-variants-c2a`). Document variants: C# resolves them (`cultcache.store.v2`); every other runtime, CultMesh single-file reads, the directory store and CultNet refuse a variant by name diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index 641cc0ed..e0026b8e 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -9,9 +9,13 @@ ### Breaking +- A single-file store holding two records under one key is unreadable: the v1 reader and the legacy envelope-array reader refuse it with `StoreUnreadableError` (a key is compared by its exact code points, no case folding or normalisation). A catalog slot whose canonical schema is not a string is refused the same way. A store that loaded before can now be refused: repair it with the runtime that wrote the duplicate. +- A `put` or `push_all` that would land a second type under a key the store holds is refused with `SchemaConflictError`, and the file is left as it was. Retype a key by deleting it, then putting the new type. A write replaces the record at a key only when the written type owns it: its catalog entry declares the stored schema id (its own, or one it lists compatible), so a renamed type that lists the old id compatible may replace the record, or it has the stored record's schema name. This runtime does not compare shapes, so a same-name type is never refused for dropping persisted slots as C# refuses it. +- Writes of the same type replace by key: the last record wins, in a batch as in the file. + - `indexes` are unique, as C# `[CultIndex]`: a write, load or `register_index` that leaves two records of a type holding one non-blank value under one alias is refused (`CultCacheError`) naming both keys and the alias, and writes nothing. Index text is the C# canonical text (`True`/`False`, integers as digits, floats as the exact expansion, blank values never indexed), so an empty or whitespace string is no longer a value. A new `roles` map names the many-row selection aliases (never indexed for lookup, same catalog `index_alias`). Classify every `indexes` key per the README before updating. The verdict is the contract, the text is informational. -- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `push_all` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. - Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload. diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index ab586ff6..236009ed 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -139,6 +139,10 @@ used by the TypeScript, Rust, and C# runtimes: Large corpora should use a sharded store or a real database. A single snapshot file is a scalpel, not a forklift. +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-py from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) + ## DatabaseEntry Slot Contract For cross-runtime cache entries, use `define_database_entry_type(...)` instead diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index badfadb5..9c402968 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -317,7 +317,7 @@ class CultCache: structures, running every check and every user extractor; the cache's state is replaced only once that build finishes, so a refused load admits nothing.""" loaded_values: dict[str, dict[str, Any]] = {} - read_order: dict[tuple[str, str], Any] = {} # (type, key) -> value, in the order the stores yielded them + read_values: dict[tuple[str, str], Any] = {} # (type, key) -> value, one per key, the last read wins; the judge is order-free loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {} loaded_legacy_global_keys: dict[str, str] = {} routes, generic = self._state.stores_by_type, self._state.generic_store @@ -356,10 +356,10 @@ class CultCache: envelope = self._stamped(document, envelope) value = document.decode_payload(envelope.payload) loaded_values.setdefault(envelope.type, {})[envelope.key] = value - read_order[(envelope.type, envelope.key)] = value + read_values[(envelope.type, envelope.key)] = value loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups( - [(type, key, value) for (type, key), value in read_order.items()], self._state.name_extractors, self._state.index_extractors + [(type, key, value) for (type, key), value in read_values.items()], self._state.name_extractors, self._state.index_extractors ) # Plain attribute assignments: nothing fallible runs once the swap begins. ( @@ -627,7 +627,7 @@ class CultCache: name_extractors: dict[str, str | Any], index_extractors: dict[str, dict[str, str | Any]], ) -> tuple[_Names, _Indexes, _LookupKeysByType]: - """Builds name and index lookups for the given (type, key, value) records, judged in the order given, in fresh dicts, touching no cache state.""" + """Builds name and index lookups for the given (type, key, value) records, judged whole and order-free, in fresh dicts, touching no cache state.""" names: _Names = {} indexes: _Indexes = {} lookup_keys: _LookupKeysByType = {} @@ -636,7 +636,7 @@ class CultCache: keys = self._lookup_keys_for(value, name_extractors.get(type), index_extractors.get(type, {})) lookup_keys.setdefault(type, {})[key] = keys landing.append((type, key, keys)) - # Judged whole, over every type at once and on the file alone (never against what the cache held): a held value can + # Judged whole and order-free, over every type at once and on the file alone (never against what the cache held): a held value can # have been mutated in place since it was admitted, and another process may have moved a value between records. self._refuse_index_sharing(landing, {}, ()) for type in lookup_keys: @@ -653,28 +653,35 @@ class CultCache: held: dict[tuple[str, str, str], str], replaced: Iterable[tuple[str, str]], ) -> None: - """The one index rule (variants:ruling:unique-indexes), in the order of C# RefuseIndexSharing: within one type and one - unique alias, no two records hold one non-blank value, judged on the result. The holders of a value are the landing - keys projecting it (in landing order) plus the held key unless this admission replaces it; two distinct holders - refuse, naming the two smallest keys (UTF-16 code-unit order) and the alias, never the value. Aliases are judged in - UTF-16 order, values in the order they landed.""" + """The one index rule (variants:ruling:unique-indexes): within one type and one unique alias, no two records hold one + non-blank value, judged on the state the admission would leave and never on staging, file or pull order. The holders + of a value are the landing keys projecting it plus the held key unless this admission replaces it. Of every + (type, alias, value) with two or more distinct holders, the refusal names the least by (alias, its least key, its + second-least key, schema name), each compared by UTF-16 code unit, never the value.""" released = set(replaced) landed: dict[tuple[str, str, str], list[str]] = {} for type, key, keys in landing: for index, index_value in keys.indexes.items(): landed.setdefault((type, index, index_value), []).append(key) - for (type, index, index_value), keys in sorted(landed.items(), key=lambda item: _utf16_order(item[0][1])): + violations: list[tuple[bytes, bytes, bytes, bytes, str, str, str, str]] = [] + for (type, index, index_value), keys in landed.items(): held_key = held.get((type, index, index_value)) if held_key is not None and (type, held_key) not in released: keys = [*keys, held_key] - distinct = sorted(dict.fromkeys(keys), key=_utf16_order) + distinct = sorted(keys, key=_utf16_order) if len(distinct) > 1: document = self._state.documents.get(type) schema_name = (document.schema_name if document is not None else None) or type - raise CultCacheError( - f"Records {distinct[0]} and {distinct[1]} ({schema_name}) both hold unique index '{index}'; " - "an index value names one record." + violations.append( + (_utf16_order(index), _utf16_order(distinct[0]), _utf16_order(distinct[1]), _utf16_order(schema_name), + index, distinct[0], distinct[1], schema_name) ) + if violations: + *_, index, first, second, schema_name = min(violations, key=lambda violation: violation[:4]) + raise CultCacheError( + f"Records {first} and {second} ({schema_name}) both hold unique index '{index}'; " + "an index value names one record." + ) def _remove_lookups(self, type: str, key: str) -> None: """Removes the lookups the held record produced at admission, handing a name to another held diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 77d671ad..e0cfdd7a 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -245,11 +245,44 @@ def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultC return [entry for entry, _ in chosen.values()] +def _retyped(held: CultCacheEnvelope, written: CultCacheEnvelope) -> SchemaConflictError: + """The refusal of a write that would put a second type under a key the store already holds (C# CacheBackingStore.Retyped).""" + return SchemaConflictError( + _schema_id_for(held), + [held.type], + held.key, + message=( + f"Record '{held.key}' is a {held.type} (schema id '{_schema_id_for(held)}'); writing a {written.type} there would " + "retype the key. Remove the record first, in the same batch or an earlier commit, then add the other type's." + ), + ) + + +def _written_owns(held: CultCacheEnvelope, written: CultCacheEnvelope) -> bool: + """Whether a write of `written` may replace `held` at its key (C# WrittenOwns): the written type's entry declares the held + record's schema id (its own id or one it lists compatible), or it is the type of the same schema name. The same-name half + stands in for C#'s "reads the published entry whole": this runtime does not compare shapes, so it never refuses a + same-name type for dropping slots.""" + stored_id = _schema_id_for(held) + entry = written.catalog_entry if written.catalog_entry is not None else _default_catalog_entry(written) + return stored_id == entry.schema_id or stored_id in entry.compatible_schema_ids or held.type == written.type + + def _encode_snapshot( supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str ) -> list[Any]: - replaced = {(envelope.type, envelope.key) for envelope in supplied} - kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced] + # A key names one record whatever its type, so a supplied record replaces the arrived one by key alone, and only of the + # type that owns it: retyping a key is a removal and an add, so a record another type owns under the key refuses the write. + held_by_key = {envelope.key: envelope for envelope in arrived} + landing: dict[str, CultCacheEnvelope] = {} + for envelope in supplied: + for held in (landing.get(envelope.key), held_by_key.get(envelope.key)): + if held is not None and not _written_owns(held, envelope): + raise _retyped(held, envelope) + landing[envelope.key] = envelope + supplied = list(landing.values()) + replaced = set(landing) + kept = [envelope for envelope in arrived if envelope.key not in replaced] envelopes = [*kept, *supplied] catalog = [ _encode_catalog_entry(entry) @@ -400,6 +433,7 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None catalog_by_schema_id.setdefault(compatible_schema_id, entry) envelopes: list[CultCacheEnvelope] = [] + seen: set[str] = set() for raw_record in decoded[2]: if not isinstance(raw_record, list) or len(raw_record) < 4: raise ValueError("CultCache persisted records must be MessagePack arrays") @@ -412,6 +446,9 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None key, schema_id, stored_at, payload = raw_record if not isinstance(key, str) or not key: raise ValueError("CultCache persisted records must declare a key") + if key in seen: + raise ValueError(f"CultCache record {key!r} appears more than once") + seen.add(key) if not isinstance(schema_id, str) or not schema_id: raise ValueError("CultCache persisted records must declare a schema id") if not isinstance(stored_at, str) or not stored_at: @@ -437,9 +474,13 @@ def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]: if not isinstance(decoded, list): raise ValueError("CultCache MessagePack store is not a recognized snapshot") envelopes: list[CultCacheEnvelope] = [] + seen: set[Any] = set() for item in decoded: if not isinstance(item, dict): raise ValueError("CultCache legacy envelopes must be MessagePack maps") + if item.get("key") in seen: + raise ValueError(f"CultCache record {item['key']!r} appears more than once") + seen.add(item.get("key")) try: payload = item["payload"] envelopes.append( @@ -484,12 +525,14 @@ def _decode_catalog_entry(value: Any) -> CultCacheSchemaCatalogEntry: raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid compatible schema ids') if not isinstance(members, list): raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid members') + if not isinstance(canonical_schema_json, str): + raise ValueError(f'CultCache schema catalog entry "{schema_id}" has an invalid canonical schema') return CultCacheSchemaCatalogEntry( schema_id=schema_id, schema_name=schema_name, schema_version=schema_version if isinstance(schema_version, str) and schema_version else f"{schema_name}.v1", content_hash=content_hash if isinstance(content_hash, str) and content_hash else schema_id, - canonical_schema_json=canonical_schema_json if isinstance(canonical_schema_json, str) else "", + canonical_schema_json=canonical_schema_json, compatible_schema_ids=tuple(compatible_schema_ids), members=tuple(_decode_catalog_member(member) for member in members), ) diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 92892b51..80abbd4f 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -1604,6 +1604,118 @@ class CultCacheTests(unittest.TestCase): self.assertIsNotNone(refused.exception.__cause__, f"{vector} {operation}") self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read") + @staticmethod + def _rec(type: str, key: str, second: int = 0) -> CultCacheEnvelope: + return CultCacheEnvelope(key=key, type=type, payload=b"\x90", stored_at=f"2026-09-30T00:00:0{second}Z") + + def test_a_supplied_record_replaces_an_arrived_one_of_the_same_type_by_key(self) -> None: + import msgpack # type: ignore + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + store = SingleFileMessagePackBackingStore(path) + store.push(self._rec("first", "k")) + store.push(self._rec("first", "k", 1)) + store.push_all([self._rec("first", "k", 2), self._rec("first", "k", 3)]) + records = msgpack.unpackb(path.read_bytes(), raw=False)[2] + self.assertEqual([record[0] for record in records], ["k"]) + self.assertEqual([(e.type, e.stored_at) for e in store.pull_all()], [("first", "2026-09-30T00:00:03Z")]) + + def test_a_write_that_would_retype_a_key_is_refused_and_leaves_the_file(self) -> None: + from cultcache_py import SchemaConflictError + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + store = SingleFileMessagePackBackingStore(path) + store.push(self._rec("first", "k")) + content = path.read_bytes() + for run in ( + lambda: store.push(self._rec("second", "k", 1)), + lambda: store.push_all([self._rec("other", "j"), self._rec("second", "k", 1)]), + ): + with self.assertRaises(SchemaConflictError) as refused: + run() + self.assertEqual(refused.exception.record_key, "k") + self.assertEqual(refused.exception.schema_names, ["first"]) + self.assertIn("would retype the key", str(refused.exception)) + self.assertEqual(path.read_bytes(), content) + # Retyping is a removal and an add. + store.delete("first", "k") + store.push(self._rec("second", "k", 1)) + self.assertEqual([e.type for e in SingleFileMessagePackBackingStore(path).pull_all()], ["second"]) + + def test_a_retype_is_judged_by_the_stored_schema_id_the_written_type_declares_not_by_name(self) -> None: + from cultcache_py import SchemaConflictError + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + store = SingleFileMessagePackBackingStore(path) + store.push(self._writer_record("k", "old", "id-old", self._writer_entry("id-old", "old", "h1", ("id-old",)))) + content = path.read_bytes() + # A renamed type that lists the stored id compatible owns the record, so its write replaces it. + renamed = self._writer_record("k", "new", "id-new", self._writer_entry("id-new", "new", "h2", ("id-new", "id-old"))) + # A type that lists nothing of the stored id and has another name does not. + stranger = self._writer_record("k", "stranger", "id-x", self._writer_entry("id-x", "stranger", "h3", ("id-x",))) + for run in (lambda: store.push(stranger), lambda: store.push_all([stranger])): + with self.assertRaises(SchemaConflictError): + run() + self.assertEqual(path.read_bytes(), content) + store.push(renamed) + self.assertEqual([(e.type, e.schema_id) for e in SingleFileMessagePackBackingStore(path).pull_all()], [("new", "id-new")]) + + def test_a_batch_naming_one_key_under_two_types_is_refused_whole(self) -> None: + from cultcache_py import SchemaConflictError + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + store = SingleFileMessagePackBackingStore(path) + with self.assertRaises(SchemaConflictError): + store.push_all([self._rec("first", "k"), self._rec("second", "k", 1)]) + self.assertFalse(path.exists()) + + def test_a_cache_put_of_a_second_type_under_a_key_is_refused_and_persists_nothing_new(self) -> None: + from cultcache_py import SchemaConflictError + + other = define_document_type("other", encode=lambda item: asdict(item), decode=lambda raw: Item(**raw)) + document = item_doc() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + + def build() -> CultCache: + cache = ( + CultCache.builder() + .register_document_type(document) + .register_document_type(other) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + cache.pull_all_backing_stores() + return cache + + cache = build() + cache.put(document, "shared", Item(name="A", category="c", value=1)) + content = path.read_bytes() + with self.assertRaises(SchemaConflictError): + cache.put(other, "shared", Item(name="B", category="c", value=2)) + self.assertEqual(path.read_bytes(), content) + self.assertIsNone(cache.get(other, "shared")) + reloaded = build() + self.assertEqual(reloaded.get_required(document, "shared").value, 1) + self.assertIsNone(reloaded.get(other, "shared")) + cache.delete(document, "shared") + cache.put(other, "shared", Item(name="B", category="c", value=2)) + self.assertEqual(build().get_required(other, "shared").value, 2) + + def test_a_catalog_entry_with_a_non_string_canonical_schema_is_unreadable(self) -> None: + # Direct probe of the c0 vector; its manifest row is added by the TS cut. + vector = self._C2A_VECTORS.parent / "document-variants-c0" / "current-catalog-bad-canonical.msgpack" + content = vector.read_bytes() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(content) + with self.assertRaises(StoreUnreadableError): + SingleFileMessagePackBackingStore(path).pull_all() + self.assertEqual(path.read_bytes(), content) if __name__ == "__main__": diff --git a/packages/cultcache-py/tests/test_unique_index.py b/packages/cultcache-py/tests/test_unique_index.py index 2d988e78..accc27dd 100644 --- a/packages/cultcache-py/tests/test_unique_index.py +++ b/packages/cultcache-py/tests/test_unique_index.py @@ -470,6 +470,76 @@ class VectorTests(unittest.TestCase): self.assertEqual(path.read_bytes(), before) + def test_refusals(self) -> None: + ids = { + "tests.unique_item": "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", + "tests.uq_pair": "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", + "tests.uq_alias": "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", + "tests.uq_wide": "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", + "tests.uq_alpha": "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", + "tests.uq_beta": "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", + } + aliases = { + "tests.uq_pair": {"Sku": "sku", "Alt": "second"}, + "tests.uq_alias": {"Sku": "sku", "alt": "second"}, + "tests.uq_wide": {"\uff21": "sku", "\U00010400": "second"}, # U+FF21 and U+10400: code-unit order puts the astral alias first + } + documents = { + name: define_database_entry_type( + name, + [database_entry_field("sku", 0, codec=t.string), database_entry_field("second", 1, codec=t.string)], + indexes=aliases.get(name, {"Sku": "sku"}), + roles={}, + schema_id=schema_id, + schema_name=name, + ) + for name, schema_id in ids.items() + } + rows = msgpack.unpackb((VECTORS / "refusals.msgpack").read_bytes(), raw=False) + walked = 0 + for label, ops, expected in rows: + if any(op[0] == "commit" and len({schema for _, schema, _, _ in op[1]}) > 1 for op in ops): + continue # put_envelopes takes one type: no batch across types (rows 3 and 26) + with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.cc" + state: dict[str, CultCache] = {} + + def perform(op) -> None: + if op[0] == "open": + if op[1] is not None: + shutil.copyfile(VECTORS / op[1], path) + state["cache"] = cache_of(*documents.values(), store=SingleFileMessagePackBackingStore(path)) + state["cache"].pull_all_backing_stores() + elif op[0] == "pull": + shutil.copyfile(VECTORS / op[1], path) + state["cache"].pull_all_backing_stores() + else: + document = documents[op[1][0][1]] + state["cache"].put_envelopes( + document, [envelope(document, key, {"sku": sku, "second": second}) for key, _, sku, second in op[1]] + ) + + for op in ops[:-1]: + perform(op) + if expected is None: + perform(ops[-1]) + else: + last = ops[-1] + if last[0] == "open": + before = (VECTORS / last[1]).read_bytes() + elif last[0] == "pull": + before = (VECTORS / last[1]).read_bytes() + else: + before = path.read_bytes() if path.exists() else b"" + k1, k2, schema, alias = expected + with self.assertRaises(CultCacheError) as caught: + perform(last) + self.assertEqual(str(caught.exception), refusal(k1, k2, schema, alias)) + self.assertEqual(path.read_bytes() if path.exists() else b"", before) + walked += 1 + self.assertEqual(walked, 35) + + class ListStore(RecordingStore): """A store whose contents the test swaps between loads: another process rewrote the file.""" @@ -515,22 +585,6 @@ class BatchAndOrderTests(unittest.TestCase): ) self.assertNotIn("item:j", cache.snapshot().get("item", {})) - def test_values_are_named_in_landing_order(self) -> None: - document = item_doc() - cache = cache_of(document) - rows = [("item:a", "B"), ("item:b", "B"), ("item:x", "A"), ("item:y", "A")] - with self.assertRaises(CultCacheError) as caught: - cache.put_envelopes(document, [envelope(document, key, Item(sku=sku)) for key, sku in rows]) - self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "sku")) - - def test_keys_sort_by_utf16_code_unit_not_code_point(self) -> None: - document = item_doc() - cache = cache_of(document) - astral, bmp = "k\U0001F600", "k\uff01" # code points: bmp < astral; UTF-16 units: D83D < FF01 - with self.assertRaises(CultCacheError) as caught: - cache.put_envelopes(document, [envelope(document, bmp, Item(sku="S")), envelope(document, astral, Item(sku="S"))]) - self.assertEqual(str(caught.exception), refusal(astral, bmp, "item", "sku")) - def test_a_load_judges_every_type_at_once_aliases_in_ordinal_order(self) -> None: first = item_doc("first", indexes={"zeta": "sku"}, roles={}) second = item_doc("second", indexes={"alpha": "sku"}, roles={}) @@ -553,7 +607,7 @@ class BatchAndOrderTests(unittest.TestCase): cache.put_envelopes(document, [envelope(document, f"item:{n}", row) for n, row in enumerate(rows)]) self.assertEqual(str(caught.exception), refusal("item:0", "item:1", "item", astral)) - def test_a_load_judges_records_in_the_order_the_file_yields_them(self) -> None: + def test_a_load_names_the_least_violation_whatever_order_the_store_yields(self) -> None: alpha, beta = item_doc("alpha", indexes={"k": "sku"}, roles={}), item_doc("beta", indexes={"k": "sku"}, roles={}) loose_alpha, loose_beta = item_doc("alpha", indexes={}, roles={}), item_doc("beta", indexes={}, roles={}) store = ListStore( @@ -568,7 +622,7 @@ class BatchAndOrderTests(unittest.TestCase): cache = cache_of(alpha, beta, store=store) with self.assertRaises(CultCacheError) as caught: cache.pull_all_backing_stores() - self.assertEqual(str(caught.exception), refusal("b:1", "b:2", "beta", "k")) + self.assertEqual(str(caught.exception), refusal("a:1", "a:2", "alpha", "k")) def test_register_index_refuses_an_alias_the_definition_declares_as_a_role(self) -> None: document = item_doc() diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index f136bccf..c32773d9 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -8,12 +8,16 @@ only in git history. ### Breaking -- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). - `#[cultcache(index)]` means unique, as in C# `RefuseIndexSharing`: within one type and one alias no two records hold one non-blank value. A put, batch, push or load that would make two records hold a value is refused, naming both keys and the alias, and a store that already holds a duplicate does not open. A grouping index is no longer expressible; a consumer that relied on shared values must drop the `index` and look up by another way. - `bool` index and lookup text renders `True` / `False`, as C# does, where it rendered `true` / `false`. - `DatabaseEntry::UNIQUE_INDEXES` replaces `cult_indexes` as the one index declaration and changes type (its members are the unique aliases); a hand implementation overriding `cult_indexes` no longer compiles. - `register_entry_type` over a type that already holds records re-judges them under the new declaration (unique indexes and the global rule) and refuses, changing nothing, when they violate it. A declaration with an index also refuses when a held record cannot be decoded as the new type; a declaration without one does not read the records, so it does not. +- A single-file store holds one record per key. The v1 and legacy readers refuse a store with two records under one key (`CultCacheStoreUnreadable`, the record `Undecodable`): the store does not open, and a write over it refuses and leaves the file. A consumer that holds such a file must repair it with the runtime that wrote it before opening it here. +- Every single-file writer (`push`, `push_all`, `insert_entry_if_absent`, the compare-exchange and snapshot-append family, and the cache's puts and batches) refuses to put a second type under a key the store holds, with `SchemaConflictError` naming the key and both types, writing nothing. Retyping a key is a delete, then an add (one batch may hold both). A write owns the held record when its schema id equals the id the held record is stored under, as C# `WrittenOwns` asks; a type renamed under a stable id therefore replaces its own record, and a write of another type name under another id is refused (`push` of the same type name under another id still replaces the record, as before; `compare_and_swap_entry`, and a `push_all` batch holding one type under two ids, refuse it). Registered types here declare no compatible ids, so equality is the whole test. Writers used to leave two records under the key. +- `compare_and_swap_entry` over a single file refuses an in-place retype (expected `A/k`, replacement `B/k` under another schema id) with the same `SchemaConflictError`, as `compare_exchange` does, where it used to retype the record and return `true`. A lost race still returns `false` first. +- A write of the same type under a held key replaces the record by key, and in one `push_all` the last given of several entries under a key and a schema id is the record, whatever their type names. The redb stores are unchanged and still key by type and key. ### Added diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 1dab4fc7..45724843 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -394,6 +394,10 @@ fn outcome(error: &anyhow::Error) { } ``` +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-rs from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) + ## Near-Term Ergonomic Improvements 1. **Derive macro** diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 11d21330..868589a1 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -143,6 +143,8 @@ use serde::de::DeserializeOwned; use std::any::Any; use std::collections::BTreeMap; use std::collections::BTreeSet; +use std::collections::HashMap; +use std::collections::HashSet; use std::fs; use std::fs::File; use std::fs::OpenOptions; @@ -929,7 +931,8 @@ fn store_header(bytes: &[u8]) -> StoreHeader { } /// A write the catalog cannot describe: records of different types under one schema id. This writer derives one entry per carried -/// schema id from its records, so that is its only conflict. Nothing is written. It names the id, the two names and a record key. +/// schema id from its records, so that is its only conflict besides a single-file write that would put a second type under a +/// key the store already holds (the key is the record's identity there). Nothing is written. It names the id, the two names and a record key. #[derive(Debug)] pub struct SchemaConflictError { pub schema_id: String, @@ -1184,6 +1187,10 @@ impl SingleFileMessagePackBackingStore { if entries[index] != *expected { return Ok(false); } + // An in-place retype is a retype: the replacement must own the schema id the held record is stored under. + if schema_identity(&entries[index]) != schema_identity(&replacement) { + return Err(retype_conflict(&entries[index], &replacement)); + } entries[index] = replacement; entries.sort_by_key(entry_id); self.write_all_unlocked(&entries)?; @@ -1491,7 +1498,8 @@ impl SingleFileMessagePackBackingStore { Err(error) if io_not_found(&error) => DiskStore::absent(), other => other?, }; - let bytes = encode_store_snapshot(entries, &disk) + let entries = settle_one_record_per_key(entries, &disk)?; + let bytes = encode_store_snapshot(&entries, &disk) .and_then(|snapshot| rmp_serde::to_vec(&snapshot).context("failed to encode MessagePack")) .map_err(|error| error.context(failed(CultCacheStoreWriteFailedKind::Rejected)))?; self.replace_unlocked(&bytes) @@ -1745,6 +1753,7 @@ impl CacheBackingStore for SingleFileMessagePackBackingStore { fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()> { self.with_exclusive_lock(|| { let mut entries = self.read_all_unlocked()?; + // A write replaces the record of its own type under its key; a second type under the key reaches the commit guard. entries.retain(|candidate| entry_id(candidate) != entry_id(entry)); entries.push(entry.clone()); entries.sort_by_key(entry_id); @@ -1768,7 +1777,13 @@ impl CacheBackingStore for SingleFileMessagePackBackingStore { return Ok(()); } - let mut entries = entries.to_vec(); + // Of entries sharing a key and a schema id the last one given is the record, whatever their type names (a type + // renamed under a stable id); a second schema id under a key reaches the commit guard. + let mut by_id = BTreeMap::<(String, String), &CultCacheEnvelope>::new(); + for entry in entries { + by_id.insert((schema_identity(entry).to_string(), entry.key.clone()), entry); + } + let mut entries: Vec = by_id.into_values().cloned().collect(); entries.sort_by_key(entry_id); self.write_all_unlocked(&entries) }) @@ -2738,6 +2753,16 @@ pub struct CultCache { } type IndexProjector = fn(&[u8]) -> Result>; +/// Orders by UTF-16 code unit (C# `StringComparer.Ordinal`), not by UTF-8 bytes or code point. +fn utf16_cmp(left: &str, right: &str) -> std::cmp::Ordering { + left.encode_utf16().cmp(right.encode_utf16()) +} + +/// Orders violations (alias, least key, second key, schema) field by field, each by UTF-16. +fn violation_cmp(left: &[&str; 4], right: &[&str; 4]) -> std::cmp::Ordering { + left.iter().zip(right).map(|(a, b)| utf16_cmp(a, b)).find(|order| order.is_ne()).unwrap_or(std::cmp::Ordering::Equal) +} + type IndexSlot = (String, String, String); type IndexHolders = BTreeMap; @@ -3424,22 +3449,36 @@ impl CultCache { groups[index].1.insert(key); } } - groups.sort_by(|left, right| left.0.1.cmp(&right.0.1)); - for (slot, mut holders) in groups { - if let Some(held_key) = held.get(&slot) + let mut least: Option<[&str; 4]> = None; + for (slot, holders) in &mut groups { + if let Some(held_key) = held.get(&*slot) && !last.contains_key(&(slot.0.as_str(), held_key.as_str())) { holders.insert(held_key.clone()); } - let mut ordered = holders.iter(); - if let (Some(first), Some(second)) = (ordered.next(), ordered.next()) { - return Err(anyhow!( - "Records {first} and {second} ({}) both hold unique index '{}'; an index value names one record.", - self.definitions.get(&slot.0).copied().unwrap_or(&slot.0), - slot.1 - )); + } + let mut ordered: Vec<(&IndexSlot, Vec<&String>)> = Vec::new(); + for (slot, holders) in &groups { + let mut keys: Vec<&String> = holders.iter().collect(); + keys.sort_by(|left, right| utf16_cmp(left, right)); + ordered.push((slot, keys)); + } + for (slot, keys) in &ordered { + if let [first, second, ..] = keys.as_slice() { + let schema = self.definitions.get(&slot.0).copied().unwrap_or(&slot.0); + let violation = [slot.1.as_str(), first.as_str(), second.as_str(), schema]; + if least.is_none_or(|best| violation_cmp(&violation, &best).is_lt()) { + least = Some(violation); + } } - let key = holders.into_iter().next().expect("a group holds at least one key"); + } + if let Some([alias, first, second, schema]) = least { + return Err(anyhow!( + "Records {first} and {second} ({schema}) both hold unique index '{alias}'; an index value names one record." + )); + } + for (slot, mut keys) in ordered.into_iter().map(|(slot, keys)| (slot.clone(), keys)) { + let key = keys.remove(0).clone(); change.taken.push((slot, key)); } Ok(change) @@ -3685,7 +3724,65 @@ impl DiskStore { /// The legacy envelope array: an array of envelope maps. Its acceptance is the one verdict on an array whose first slot /// is not a store header, for reading it and for replacing it. fn decode_legacy_envelopes(bytes: &[u8]) -> Result> { - decode_grown(bytes).context("failed to decode legacy CultCache envelope array") + let envelopes: Vec = + decode_grown(bytes).context("failed to decode legacy CultCache envelope array")?; + require_unique_keys(&envelopes)?; + Ok(envelopes) +} + +/// A record's identity in a store is its key string alone, compared ordinal: a store holding a key twice is unreadable, +/// whatever the format of the file that holds it. +fn require_unique_keys(envelopes: &[CultCacheEnvelope]) -> Result<()> { + let mut keys = HashSet::<&str>::new(); + for envelope in envelopes { + if !keys.insert(envelope.key.as_str()) { + bail!("CultCache record {:?} appears more than once", envelope.key); + } + } + Ok(()) +} + +/// What a record's stored schema id is: the id it carries, else its type (a registered type's id is its entry type). This +/// runtime's registered types declare no compatible ids, so a written record owns a held one exactly when the ids are equal: +/// the question C# `WrittenOwns` asks of the stored id, not a comparison of type names. +fn schema_identity(entry: &CultCacheEnvelope) -> &str { + entry.schema_id.as_deref().unwrap_or(&entry.r#type) +} + +/// The refusal of a write that would retype a key: the written record does not own the schema id the held one is stored under. +/// Retyping is remove-then-add (variants:ruling:cache-refuses-retype). +fn retype_conflict(held: &CultCacheEnvelope, written: &CultCacheEnvelope) -> anyhow::Error { + anyhow::Error::new(SchemaConflictError { + schema_id: schema_identity(written).to_string(), + schema_names: vec![held.r#type.clone(), written.r#type.clone()], + record_key: written.key.clone(), + }) +} + +/// The commit guard every single-file write passes: a store holds one record per key. Two entries under a key whose schema +/// ids differ are a retype and are refused with the key and both types, nothing written. Two under one schema id are one +/// record: a written entry replaces the held one it owns (a type renamed under a stable id), the held one being what the file +/// already carries. +fn settle_one_record_per_key(entries: &[CultCacheEnvelope], disk: &DiskStore) -> Result> { + let arrived: BTreeMap<(String, String), &CultCacheEnvelope> = disk.envelopes.iter().map(|entry| (entry_id(entry), entry)).collect(); + let held = |entry: &CultCacheEnvelope| arrived.get(&entry_id(entry)) == Some(&entry); + let mut settled = Vec::<&CultCacheEnvelope>::new(); + let mut at_key = HashMap::<&str, usize>::new(); + for entry in entries { + match at_key.get(entry.key.as_str()) { + None => { + at_key.insert(entry.key.as_str(), settled.len()); + settled.push(entry); + } + Some(&slot) if schema_identity(settled[slot]) != schema_identity(entry) => return Err(retype_conflict(settled[slot], entry)), + Some(&slot) => { + if !held(entry) || held(settled[slot]) { + settled[slot] = entry; + } + } + } + } + Ok(settled.into_iter().cloned().collect()) } /// Decodes a store whose header the caller has already read as a readable one: its envelopes and its catalog. @@ -3724,6 +3821,7 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result<(Vec, Vec>>()?; + require_unique_keys(&envelopes)?; Ok((envelopes, snapshot.1)) } @@ -4195,32 +4293,159 @@ mod tests { Ok(()) } + fn settings_one() -> Settings { + Settings { theme: "green".to_string(), retries: 1 } + } + + fn note_one() -> Note { + Note { title: "same key".to_string(), body: "different type".to_string() } + } + + fn retype_refusal(error: &anyhow::Error) -> &SchemaConflictError { + error.downcast_ref::().unwrap_or_else(|| panic!("{error:#}")) + } + + fn reloaded(path: &Path) -> Result> { + SingleFileMessagePackBackingStore::new(path).pull_all() + } + #[test] - fn entry_identity_is_polymorphic_by_type_and_key() -> Result<()> { + fn a_second_type_under_a_stored_key_is_refused_and_the_store_and_cache_stay() -> Result<()> { let temp = tempfile::tempdir()?; let store_path = temp.path().join("cache.cc"); let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; - cache.put( - "shared", - &Settings { - theme: "green".to_string(), - retries: 1, - }, - )?; - cache.put( - "shared", - &Note { - title: "same key".to_string(), - body: "different type".to_string(), - }, - )?; - - assert_eq!(cache.snapshot().len(), 2); - assert_eq!(cache.get_required::("shared")?.title, "same key"); + cache.put("shared", &settings_one())?; + let before = std::fs::read(&store_path)?; + let error = cache.put("shared", ¬e_one()).unwrap_err(); + assert_eq!(retype_refusal(&error).record_key, "shared"); + assert_eq!(std::fs::read(&store_path)?, before, "the refused write changed the file"); + assert_eq!(cache.snapshot().len(), 1); assert_eq!(cache.get_required::("shared")?.theme, "green"); + assert!(cache.get::("shared")?.is_none()); + + // Retyping is remove-then-add, and what was written reads back. + assert!(cache.delete::("shared")?); + cache.put("shared", ¬e_one())?; + let held = reloaded(&store_path)?; + assert_eq!(held.len(), 1); + assert_eq!(held[0].r#type, Note::TYPE); + + // The same type under the key replaces it. + cache.put("shared", &Note { title: "again".to_string(), body: String::new() })?; + let held = reloaded(&store_path)?; + assert_eq!(held.len(), 1); + assert_eq!(cache.get_required::("shared")?.title, "again"); + Ok(()) + } + + #[test] + fn a_batch_holding_a_second_type_under_a_key_is_refused_whole() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("batch.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.put("kept", &settings_one())?; + let before = std::fs::read(&store_path)?; + + let (settings, _) = cache.prepare_entry("shared", &settings_one())?; + let (note, _) = cache.prepare_entry("shared", ¬e_one())?; + let error = cache.put_prepared_batch(vec![settings.clone(), note.clone()]).unwrap_err(); + assert_eq!(retype_refusal(&error).record_key, "shared"); + assert_eq!(std::fs::read(&store_path)?, before); + assert_eq!(cache.snapshot().len(), 1); + + // The store's own batch write refuses it too, in either order. + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + for batch in [[settings.clone(), note.clone()], [note, settings]] { + let error = store.push_all(&batch, PushAllOptions::default()).unwrap_err(); + assert_eq!(retype_refusal(&error).record_key, "shared"); + assert_eq!(std::fs::read(&store_path)?, before); + } + Ok(()) + } + + #[test] + fn the_stores_own_push_refuses_a_second_type_under_a_key() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("push.cc"); + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + let (settings, _) = cache.prepare_entry("shared", &settings_one())?; + let (note, _) = cache.prepare_entry("shared", ¬e_one())?; + let mut store = SingleFileMessagePackBackingStore::new(&store_path); + store.push(&settings)?; + let before = std::fs::read(&store_path)?; + let error = store.push(¬e).unwrap_err(); + let mut names = retype_refusal(&error).schema_names.clone(); + names.sort(); + assert_eq!(names, vec![Note::TYPE.to_string(), Settings::TYPE.to_string()]); + assert_eq!(std::fs::read(&store_path)?, before); + Ok(()) + } + + #[test] + fn no_single_file_writer_commits_a_second_type_under_a_held_key() -> Result<()> { + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + let (held, _) = cache.prepare_entry("k", &settings_one())?; + let (other_key, _) = cache.prepare_entry("c", &settings_one())?; + let (note, _) = cache.prepare_entry("k", ¬e_one())?; + let absent_note = CultCacheExpectedEnvelope { key: "k".to_string(), r#type: Note::TYPE.to_string(), current: None }; + type Writer = fn(&mut SingleFileMessagePackBackingStore, &[CultCacheEnvelope; 3], &CultCacheExpectedEnvelope) -> Result; + let writers: Vec<(&str, Writer)> = vec![ + ("push", |s, e, _| s.push(&e[2]).map(|()| true)), + ("push_all", |s, e, _| s.push_all(&[e[0].clone(), e[2].clone()], PushAllOptions::default()).map(|()| true)), + ("insert_entry_if_absent", |s, e, _| s.insert_entry_if_absent(e[2].clone())), + ("compare_and_swap_entry", |s, e, _| s.compare_and_swap_entry(&e[0], e[2].clone())), + ("compare_exchange", |s, e, x| s.compare_exchange(std::slice::from_ref(x), &[e[2].clone()])), + ("compare_and_swap_batch", |s, e, _| s.compare_and_swap_batch(&[], vec![e[2].clone()])), + ("append_if_snapshot_unchanged", |s, e, _| s.append_if_snapshot_unchanged(&[e[0].clone(), e[1].clone()], vec![e[2].clone()])), + ("replace_and_append_if_snapshot_unchanged", |s, e, _| { + s.replace_and_append_if_snapshot_unchanged(&[e[0].clone(), e[1].clone()], vec![e[2].clone()]) + }), + ("replace_and_delete_if_snapshot_unchanged", |s, e, _| { + s.replace_and_delete_if_snapshot_unchanged(&[e[0].clone(), e[1].clone()], vec![e[2].clone()], std::slice::from_ref(&e[1])) + }), + ("compare_exchange_snapshot", |s, e, _| s.compare_exchange_snapshot(&[e[1].clone(), e[0].clone()], &[e[0].clone(), e[2].clone()])), + ("try_compare_exchange_snapshot", |s, e, _| { + s.try_compare_exchange_snapshot(&[e[1].clone(), e[0].clone()], &[e[0].clone(), e[2].clone()]).map(|_| true) + }), + ]; + let envelopes = [held.clone(), other_key.clone(), note]; + for (name, write) in writers { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + store.push_all(&[held.clone(), other_key.clone()], PushAllOptions::default())?; + let before = std::fs::read(&path)?; + let error = write(&mut store, &envelopes, &absent_note).expect_err(name); + assert_eq!(retype_refusal(&error).record_key, "k", "{name}"); + assert_eq!(std::fs::read(&path)?, before, "{name} changed the file"); + assert!(store.pull_all().is_ok(), "{name}"); + } + Ok(()) + } + + #[test] + fn a_legacy_array_with_a_key_under_two_types_is_not_decodable() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("legacy.cc"); + let record = |kind: &str| { + CultCacheEnvelope { + key: "a".to_string(), + r#type: kind.to_string(), + payload: rmp_serde::to_vec(&Vec::::new()).unwrap(), + stored_at: "2026-01-01T00:00:00Z".to_string(), + schema_id: None, + } + }; + std::fs::write(&store_path, rmp_serde::to_vec_named(&vec![record("old.one"), record("old.two")])?)?; + let error = SingleFileMessagePackBackingStore::new(&store_path).pull_all().unwrap_err(); + assert_eq!(unreadable_kind(&error), Some(CultCacheStoreUnreadableKind::Undecodable), "{error:#}"); Ok(()) } @@ -5884,6 +6109,26 @@ mod tests { } + // The writer keeps one record per key: another type under a held key is refused and the file stays; the same type replaces. + #[test] + fn a_write_of_another_type_under_an_existing_key_is_refused_and_the_same_type_replaces() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + let mut other = snapshot_envelope("k", b"two"); + other.r#type = "snapshot.other".to_string(); + other.schema_id = Some("snapshot.other".to_string()); + store.push(&snapshot_envelope("k", b"one"))?; + let before = std::fs::read(&path)?; + assert!(store.push(&other).unwrap_err().downcast_ref::().is_some()); + assert!(store.push_all(&[snapshot_envelope("k", b"one"), other.clone()], PushAllOptions::default()).is_err()); + assert_eq!(std::fs::read(&path)?, before); + store.push(&snapshot_envelope("k", b"three"))?; + let read = store.pull_all()?; + assert_eq!((read.len(), read[0].payload.as_slice()), (1, b"three".as_slice())); + Ok(()) + } + // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by open, // push_all and push alike. The bytes and every runtime's verdict are tests/vectors/document-variants-c2a/readability. // A refusal is a CultCacheStoreUnreadable, and a replaced file carries the header its old content decides. @@ -6408,6 +6653,93 @@ mod tests { Ok(()) } + // The retype refusal asks what C# WrittenOwns asks: does the written record own the schema id the held one is stored under. + // A type renamed under a stable id owns it, so its write replaces the record; another id does not, and is refused. + #[test] + fn a_write_owning_the_stored_schema_id_replaces_the_record_under_a_new_name() -> Result<()> { + let envelope = |r#type: &str, schema_id: &str, payload: &[u8]| CultCacheEnvelope { + key: "a".into(), + r#type: r#type.into(), + payload: payload.to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some(schema_id.into()), + }; + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + store_file(&path, &[("t.s", "t.old")], &[("a", "t.s", b"one".to_vec())])?; + let before = std::fs::read(&path)?; + let error = SingleFileMessagePackBackingStore::new(&path).push(&envelope("t.other", "t.other", b"x")).unwrap_err(); + assert_eq!(retype_refusal(&error).record_key, "a"); + assert_eq!(std::fs::read(&path)?, before); + + SingleFileMessagePackBackingStore::new(&path).push(&envelope("t.s", "t.s", b"two"))?; + let held = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + assert_eq!(held.len(), 1); + assert_eq!((held[0].r#type.as_str(), held[0].payload.as_slice()), ("t.s", b"two".as_slice())); + Ok(()) + } + + // Where a held record and a written one share a schema id under different type names, the written one is the record + // whichever name sorts first: the writers hand settle_one_record_per_key their entries sorted by type name. + #[test] + fn a_renamed_type_replaces_the_held_record_whether_its_name_sorts_before_or_after() -> Result<()> { + for new_name in ["a.new", "z.new"] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + store_file(&path, &[("t.s", "t.old")], &[("a", "t.s", b"one".to_vec())])?; + let written = CultCacheEnvelope { + key: "a".into(), + r#type: new_name.into(), + payload: b"two".to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some("t.s".into()), + }; + SingleFileMessagePackBackingStore::new(&path).push(&written)?; + let held = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + assert_eq!(held.len(), 1, "{new_name}"); + assert_eq!(held[0].payload, b"two".to_vec(), "{new_name}"); + } + Ok(()) + } + + // In one push_all, entries under a key and a schema id with different type names resolve by the order given, not by + // the order of the names. + #[test] + fn push_all_resolves_same_id_entries_with_different_names_by_the_order_given() -> Result<()> { + let envelope = |r#type: &str, payload: &[u8]| CultCacheEnvelope { + key: "a".into(), + r#type: r#type.into(), + payload: payload.to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some("t.s".into()), + }; + for (first, second) in [("a.name", "b.name"), ("b.name", "a.name")] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + store.push_all(&[envelope(first, b"first"), envelope(second, b"second")], PushAllOptions::default())?; + let held = store.pull_all()?; + assert_eq!(held.len(), 1, "{first} then {second}"); + assert_eq!(held[0].payload, b"second".to_vec(), "{first} then {second}"); + } + Ok(()) + } + + // Of same-type entries sharing a key in one push_all the last one given is the record. + #[test] + fn push_all_keeps_the_last_of_two_same_type_entries_under_a_key() -> Result<()> { + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + let (first, _) = cache.prepare_entry("k", &settings_one())?; + let (last, _) = cache.prepare_entry("k", &Settings { theme: "red".to_string(), retries: 2 })?; + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.cc"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + store.push_all(&[first, last.clone()], PushAllOptions::default())?; + assert_eq!(store.pull_all()?.into_iter().map(|e| e.payload).collect::>(), vec![last.payload]); + Ok(()) + } + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] #[cultcache(type = "tests.stable", schema = "tests.old_name")] struct StableId { diff --git a/packages/cultcache-rs/tests/unique_index.rs b/packages/cultcache-rs/tests/unique_index.rs index d0da4175..1132b2b5 100644 --- a/packages/cultcache-rs/tests/unique_index.rs +++ b/packages/cultcache-rs/tests/unique_index.rs @@ -80,6 +80,50 @@ struct VBool { v: bool, } +/// The refusals vector's other schemas: `Sku` is unique in each; `Alt` is a second unique index in `uq_pair` (alias +/// `Alt`) and `uq_alias` (alias `alt`, sorting after `Sku` only under an ordinal compare); alpha and beta are plain. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_pair", schema = "tests.uq_pair")] +struct UqPair { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1, index = "Alt")] + alt: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_alias", schema = "tests.uq_alias")] +struct UqAlias { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1, index = "alt")] + alt: String, +} +/// Aliases U+FF21 and U+10400: UTF-16 puts the astral alias first (D801 DC00 < FF21), code-point order the other way. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_wide", schema = "tests.uq_wide")] +struct UqWide { + #[cultcache(key = 0, index = "\u{FF21}")] + sku: String, + #[cultcache(key = 1, index = "\u{10400}")] + alt: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_alpha", schema = "tests.uq_alpha")] +struct UqAlpha { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + second: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_beta", schema = "tests.uq_beta")] +struct UqBeta { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + second: String, +} + fn unique(sku: &str) -> Unique { Unique { sku: sku.into(), category: "c".into() } } @@ -339,35 +383,6 @@ fn a_batch_is_judged_whole() { assert_eq!(cache.get_by_index::("Sku", "D1").unwrap(), cache.get::("e").unwrap()); } -// The verdict and the named pair do not depend on key order, staging order, or alias declaration order. -#[test] -fn the_refusal_names_the_ordinal_pair_whatever_the_order() { - let orders: [[&str; 3]; 6] = - [["a", "b", "c"], ["a", "c", "b"], ["b", "a", "c"], ["b", "c", "a"], ["c", "a", "b"], ["c", "b", "a"]]; - for order in orders { - let mut cache = memory_cache(); - let rows: Vec<_> = order.iter().map(|key| (*key, unique("S"))).collect(); - let text = error_text(cache.put_prepared_batch(staged(&cache, &rows))); - assert_eq!(text, refusal("a", "b", "tests.unique_item", "Sku"), "{order:?}"); - assert!(skus(&cache).is_empty()); - } - // Both aliases collide in one batch: the alias sorting first is named, though `B` is declared first. - for order in [["x", "y"], ["y", "x"]] { - let mut cache = memory_cache(); - let rows: Vec<_> = order.iter().map(|key| (*key, multi(Some("s"), Some("t")))).collect(); - let text = error_text(cache.put_prepared_batch(staged(&cache, &rows))); - assert_eq!(text, refusal("x", "y", "Multi", "A")); - } - // A held key and a landing key: the pair is ordinal, not held-first. - let mut cache = memory_cache(); - cache.put("m", &unique("S")).unwrap(); - assert_eq!(error_text(cache.put("z", &unique("S"))), refusal("m", "z", "tests.unique_item", "Sku")); - assert_eq!(error_text(cache.put("a", &unique("S"))), refusal("a", "m", "tests.unique_item", "Sku")); - // A value that differs only by case or by a surrounding space is another value. - cache.put("n", &unique("s")).unwrap(); - cache.put("o", &unique("S ")).unwrap(); -} - #[test] fn a_load_of_a_duplicate_image_is_refused_and_keeps_the_prior_view() { let dir = tempfile::tempdir().unwrap(); @@ -561,6 +576,97 @@ fn vectors_stores_hold_their_verdicts() { } } +/// One op of a refusals row: `["open", file-or-nil]`, `["pull", file]` or `["commit", [[key, schema, sku, second], ...]]`. +#[derive(Deserialize)] +#[serde(untagged)] +enum Arg { + Records(Vec<(String, String, String, String)>), + File(Option), +} + +type RefusalRow = (String, Vec<(String, Arg)>, Option<(String, String, String, String)>); + +/// Rows `[label, ops, refusal]`: the pair a refusal names is a function of the state the admission would leave +/// (`tests/vectors/unique-index/README.md`, RULE), whatever the staging, file or pull order, key case or alias case. +#[test] +fn vectors_refusals_hold_the_rule() { + let bytes = std::fs::read(vectors().join("refusals.msgpack")).unwrap(); + let rows: Vec = rmp_serde::from_slice(&bytes).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut walked = 0; + for (index, (label, ops, refused)) in rows.into_iter().enumerate() { + let path = dir.path().join(format!("row-{index}.cc")); + let mut cache: Option = None; + let mut failure = None; + for (at, (kind, arg)) in ops.iter().enumerate() { + let last = at + 1 == ops.len(); + let mut before = path.exists().then(|| std::fs::read(&path).unwrap()); + let result = match (kind.as_str(), arg) { + ("open", Arg::File(file)) => { + if let Some(file) = file { + std::fs::copy(vectors().join(file), &path).unwrap(); + } + let mut opened = CultCache::new(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + let pulled = opened.pull_all_backing_stores(); + cache = Some(opened); + pulled + } + ("pull", Arg::File(Some(file))) => { + std::fs::copy(vectors().join(file), &path).unwrap(); + before = Some(std::fs::read(&path).unwrap()); + cache.as_mut().unwrap().pull_all_backing_stores() + } + ("commit", Arg::Records(records)) => { + let cache = cache.as_mut().unwrap(); + let staged: Vec<_> = records + .iter() + .map(|(key, schema, sku, second)| { + let (sku, second) = (sku.clone(), second.clone()); + match schema.as_str() { + "tests.unique_item" => cache.prepare_entry(key, &Unique { sku, category: second }).map(|(envelope, _)| envelope), + "tests.uq_pair" => cache.prepare_entry(key, &UqPair { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_alias" => cache.prepare_entry(key, &UqAlias { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_wide" => cache.prepare_entry(key, &UqWide { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_alpha" => cache.prepare_entry(key, &UqAlpha { sku, second }).map(|(envelope, _)| envelope), + "tests.uq_beta" => cache.prepare_entry(key, &UqBeta { sku, second }).map(|(envelope, _)| envelope), + other => panic!("{label}: unexpected schema {other}"), + } + .unwrap() + }) + .collect(); + cache.put_prepared_batch(staged) + } + other => panic!("{label}: unexpected op {:?}", other.0), + }; + if last { + failure = Some((result, before)); + } else { + result.unwrap_or_else(|error| panic!("{label}: op {at} should succeed: {error:#}")); + } + } + let (result, before) = failure.unwrap(); + match refused { + None => result.unwrap_or_else(|error| panic!("{label}: should succeed: {error:#}")), + Some((k1, k2, schema, alias)) => { + let error = result.err().unwrap_or_else(|| panic!("{label}: should refuse")); + assert_eq!(format!("{error:#}"), refusal(&k1, &k2, &schema, &alias), "{label}"); + if let Some(before) = before { + assert_eq!(std::fs::read(&path).unwrap(), before, "{label}: the file is byte-identical"); + } + } + } + walked += 1; + } + assert_eq!(walked, 37); +} + // A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept. #[test] fn a_second_pull_replaces_the_holders_with_the_stored_image() { diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 9850dfe4..8a4dd9c4 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -7,12 +7,17 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Breaking +- A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical text (booleans `True`/`False`, integers as digits, doubles as the exact expansion, blank values never indexed), so an empty string is no longer a value. `registerIndex` over held duplicates installs nothing. A new `roles` map names the many-row selection aliases (never indexed for lookup, same catalog `indexAlias`). The per-key migration rule is in the README; classify every key before the CultLib your tools load is updated. The refusal verdict is the contract, its text is informational. +- The single-file reader (v1 and the legacy envelope array) refuses a store that holds two records under one key as `StoreUnreadableError`; a store that opened before may now be refused. Repair it by removing the extra record with the runtime that wrote it. +- A `put`, `push` or `pushAll` that would land a second type under a held key (against the file, or within one batch) is refused with `SchemaConflictError`, and nothing is written. The written type may replace the record when its catalog entry carries the stored schema id as its own or lists it compatible (a type renamed under a stable id restamps the record, as C# does); otherwise retyping a key is a delete, then a put. The C# allowance for an undeclared older id of the same schema name that reads whole is not ported: a same-named type keeps passing, including one that drops persisted slots, which C# refuses. +- Same-type writes replace by key; in one `pushAll` batch the last entry for a key wins. +- A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. ### Changed @@ -27,7 +32,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Breaking -- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. - A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v`. - A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. A full `pushAll` over a file with no readable stamp does not replace it: the reader refuses it. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole. - A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store. diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 97fb3cdd..21e76701 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -63,6 +63,10 @@ const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. The whole cache without the single-file store is `@gamecult/cultcache-ts/browser`, which is also what bundlers resolve for the package root (`browser` condition); bring your own `CacheBackingStore`. +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads a v3 store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-ts from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) + ## `.cc` Inspection CultCache files may use the `.cc` extension. The bytes are still the canonical diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 775118d4..c5bb663d 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,4 +1,5 @@ import { + declaresRole, decodeDocumentPayload, documentPayloadBytes, documentPayloadFromBytes, @@ -46,6 +47,7 @@ type EntryLookups = { }; type HydratedEntry = CultCacheEnvelope & { + catalogEntry: NonNullable; value: unknown; lookups: EntryLookups; }; @@ -240,6 +242,10 @@ export class CultCache { throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`); } + if (declaresRole(definition, indexName)) { + throw new Error(`Index "${indexName}" of ${definition.type} is declared as a role; a role is never a unique index.`); + } + const registered = this.#requireDefinition(definition); const indexAccessors = new Map(registered.indexAccessors); indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); @@ -311,6 +317,16 @@ export class CultCache { } } + // Judged whole, once: the refusal names the least violation of the state this load would leave, whatever the + // order of the stores and of the entries in them. + refuseLeastIndexSharing(function* () { + for (const entry of next.entries.values()) { + for (const [alias, value] of entry.lookups.indexes) { + yield { type: entry.type, schemaName: entry.catalogEntry.schemaName, alias, value, key: entry.key }; + } + } + }()); + this.#hydrated = next; }); } @@ -647,6 +663,7 @@ export class CultCache { async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise { const home = this.#homeStore(entry.type); const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value); + this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups); const legacy = this.#hydrated.legacyGlobals.get(entry.type); await home?.push(entry); if (home && legacy) { @@ -898,7 +915,6 @@ export class CultCache { } const lookups = this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value); - this.#refuseIndexSharing(state, entry.type, registered.catalogEntry.schemaName, entry.key, lookups); return { ...entry, payload: this.#cloneBytes(entry.payload), @@ -990,7 +1006,6 @@ export class CultCache { const entryLookups: Array<[HydratedEntry, EntryLookups]> = []; const nameMap = nameAccessor ? new Map() : undefined; const indexMaps = new Map>(); - const sharing = new Map(); for (const indexName of indexAccessors.keys()) { indexMaps.set(indexName, new Map()); } @@ -1003,21 +1018,16 @@ export class CultCache { } for (const [indexName, indexValue] of lookups.indexes) { const map = indexMaps.get(indexName)!; - const holder = map.get(indexValue); - if (holder === undefined) { - map.set(indexValue, entry.key); - } else { - sharing.set(`${indexName}\u0000${indexValue}`, [...(sharing.get(`${indexName}\u0000${indexValue}`) ?? [holder]), entry.key]); - } + if (!map.has(indexValue)) map.set(indexValue, entry.key); } } - // Judged whole: any value held by two records refuses, naming the two smallest of its keys. Held records already - // passed the judge for every alias installed before, so only the alias being registered can be shared here. - for (const [id, keys] of sharing) { - keys.sort(compareOrdinal); - throw indexSharingRefusal(keys[0], keys[1], schemaName, id.slice(0, id.indexOf("\u0000"))); - } + // Judged whole, by the same judge as a load: the least violation of the state the registration would leave. + refuseLeastIndexSharing(function* () { + for (const [entry, lookups] of entryLookups) { + for (const [alias, value] of lookups.indexes) yield { type, schemaName, alias, value, key: entry.key }; + } + }()); return { entryLookups, nameMap, indexMaps }; } @@ -1094,6 +1104,34 @@ function compareOrdinal(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } +// The whole-state judge (variants:question:unique-index-refusal-pair-rule): over every (type, alias, value) held at two +// or more distinct keys, throws for the one least by (alias, least key, second-least key, schema name), each compared by +// UTF-16 code unit. A function of the state, never of the order the rows arrive in. +function refuseLeastIndexSharing(rows: Iterable<{ type: string; schemaName: string; alias: string; value: string; key: string }>): void { + const groups = new Map(); + for (const { type, schemaName, alias, value, key } of rows) { + const id = JSON.stringify([type, alias, value]); + const group = groups.get(id); + if (group) group.keys.push(key); + else groups.set(id, { schemaName, alias, keys: [key] }); + } + let least: { schemaName: string; alias: string; first: string; second: string } | undefined; + for (const { schemaName, alias, keys } of groups.values()) { + if (keys.length < 2) continue; + keys.sort(compareOrdinal); + const candidate = { schemaName, alias, first: keys[0], second: keys[1] }; + if (!least || compareLeast(candidate, least) < 0) least = candidate; + } + if (least) throw indexSharingRefusal(least.first, least.second, least.schemaName, least.alias); +} + +function compareLeast( + a: { schemaName: string; alias: string; first: string; second: string }, + b: { schemaName: string; alias: string; first: string; second: string }, +): number { + return compareOrdinal(a.alias, b.alias) || compareOrdinal(a.first, b.first) || compareOrdinal(a.second, b.second) || compareOrdinal(a.schemaName, b.schemaName); +} + function indexSharingRefusal(keyA: string, keyB: string, schemaName: string, alias: string): Error { const [first, second] = compareOrdinal(keyA, keyB) <= 0 ? [keyA, keyB] : [keyB, keyA]; return new Error(`Records ${first} and ${second} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`); diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index bc4ace88..10af96a6 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -131,6 +131,12 @@ export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition return documentPayloadFromBytes(bytes); } +// Whether the definition declares `name` as a role alias (either `roles` form). +export function declaresRole(definition: AnyCultCacheDocumentDefinition, name: string): boolean { + const roles = definition.roles; + return Array.isArray(roles) ? roles.some((entry) => entry.name === name) : roles ? Object.hasOwn(roles, name) : false; +} + // The schema members of a document: the ones it declares, else those its codec derives, else none. export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinition): readonly CultCacheSchemaCatalogMember[] { if (definition.members || !definition.codec) { diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 6428ef48..3f484ad8 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -60,9 +60,8 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const disk = this.#readDisk(); this.#format = disk.format; const existing = disk.envelopes; - const filtered = existing.filter( - (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), - ); + requireOneTypePerKey(this.filePath, [entry], existing); + const filtered = existing.filter((candidate) => candidate.key !== entry.key); await this.#writeAll([entry], filtered); }); } @@ -103,7 +102,9 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { this.#format = STORE_FORMAT_VERSION; } - await this.#writeAll(entries, []); + requireOneTypePerKey(this.filePath, entries, []); + // A key is held once: of same-type entries sharing a key in one batch, the last is kept. + await this.#writeAll([...new Map(entries.map((entry) => [entry.key, entry])).values()], []); }); } @@ -137,6 +138,36 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } } +// As C# WrittenOwns' declared half: the written type owns the stored schema id when its catalog entry carries that id as its own +// or lists it compatible, so a type renamed under a stable id restamps the record it already holds. (The C# half that lets an +// undeclared older id of the same schema name through when it reads whole is not ported; a same-named type keeps passing.) +function writtenOwns(written: CultCacheEnvelope, storedId: string): boolean { + const entry = written.catalogEntry; + return entry !== undefined && (entry.schemaId === storedId || (entry.compatibleSchemaIds ?? []).includes(storedId)); +} + +// A store holds a key once, and the reader refuses a file that holds it twice, so a write that would put a second type under a +// key (against a record on disk, or within its own batch) is refused before anything is written. Retyping a record is a delete +// of the old type followed by a put of the new one. +function requireOneTypePerKey(path: string, supplied: CultCacheEnvelope[], held: CultCacheEnvelope[]): void { + const typeByKey = new Map(); + for (const entry of held) { + typeByKey.set(entry.key, entry); + } + for (const entry of supplied) { + const prior = typeByKey.get(entry.key); + if (prior !== undefined && prior.type !== entry.type && !writtenOwns(entry, schemaIdForEnvelope(prior))) { + throw new SchemaConflictError( + schemaIdForEnvelope(entry), + [prior.type, entry.type], + entry.key, + `CultCache store ${path} holds record "${entry.key}" as type "${prior.type}", so type "${entry.type}" cannot be written under the same key; remove the record first.`, + ); + } + typeByKey.set(entry.key, entry); + } +} + /** * The one reader of a CultCache single-file store, for this package's store and for any consumer that must read a * store file (an authority bundle, say) rather than decode it itself. A path with nothing at it is an absent, empty, @@ -204,8 +235,9 @@ type DecodedSnapshot = { }; function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] { - const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`)); - const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`)); + // A key is held once: a supplied record replaces the arrived record under its key whatever the type. + const replaced = new Set(supplied.map((entry) => entry.key)); + const kept = arrived.filter((entry) => !replaced.has(entry.key)); const entries = [...kept, ...supplied]; const catalog = catalogEntriesFor([ ...kept.map((envelope) => ({ envelope, registered: false })), @@ -416,6 +448,7 @@ function decodeStoreFile(data: Uint8Array): Omit throw new Error("CultCache file is not a recognized CultCache MessagePack store."); } + requireUniqueKeys(legacy); const normalized = legacy.map((entry) => ({ ...entry, payload: normalizePayload(entry.payload) })); return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[] }; } @@ -449,9 +482,21 @@ function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined { } const records = recordsRaw.map(decodeRecord); + requireUniqueKeys(records); return { format: decoded[0], catalogBySchemaId, records }; } +// A store holds a key once, whatever the type: a repeat is refused in a v1 snapshot and in a legacy envelope array alike. +function requireUniqueKeys(records: Array<{ key: string }>): void { + const seen = new Set(); + for (const { key } of records) { + if (seen.has(key)) { + throw new Error(`CultCache record "${key}" appears more than once.`); + } + seen.add(key); + } +} + function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry { if (!Array.isArray(value)) { throw new Error("CultCache schema catalog entries must be MessagePack arrays."); @@ -479,12 +524,16 @@ function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry { throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid members.`); } + if (typeof canonicalSchemaJson !== "string") { + throw new Error(`CultCache schema catalog entry "${schemaId}" has an invalid canonical schema.`); + } + return { schemaId, schemaName, schemaVersion: isNonEmptyString(schemaVersion) ? schemaVersion : `${schemaName}.v1`, contentHash: isNonEmptyString(contentHash) ? contentHash : schemaId, - canonicalSchemaJson: typeof canonicalSchemaJson === "string" ? canonicalSchemaJson : "", + canonicalSchemaJson, compatibleSchemaIds, members: members.map(decodeCatalogMember), }; diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index de0cad79..2675545e 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -14,7 +14,7 @@ import { z } from "zod"; import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; -import { SingleFileMessagePackBackingStore, readStoreBytes } from "../src/single-file-messagepack-backing-store"; +import { SingleFileMessagePackBackingStore, readSingleFileStore, readStoreBytes } from "../src/single-file-messagepack-backing-store"; import { SchemaConflictError, StoreUnreadableError, refuseUnreadStoreBytes } from "../src/store-format"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; @@ -143,6 +143,63 @@ test("CultCache supports registry bootstrap, global documents, and lookup by nam }); }); +test("a put that would land a second type under a persisted key is refused, the cache and the file are unchanged, and the store reloads", async () => { + const alpha = defineDocumentType({ type: "alpha", schema: z.object({ v: z.number() }) }); + const beta = defineDocumentType({ type: "beta", schema: z.object({ v: z.number() }) }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "cache.msgpack"); + const registry = defineDocumentRegistry(alpha, beta); + const open = () => CultCache.builder().withRegistry(registry).withGenericStore(new SingleFileMessagePackBackingStore(storePath)).build(); + + const cache = open(); + await cache.put(alpha, "shared", { v: 1 }); + const before = [...(await readFile(storePath))]; + await assert.rejects(() => cache.put(beta, "shared", { v: 2 }), SchemaConflictError); + assert.deepEqual([...(await readFile(storePath))], before); + assert.deepEqual(cache.get(alpha, "shared"), { v: 1 }); + assert.equal(cache.get(beta, "shared"), undefined); + + const reloaded = open(); + await reloaded.pullAllBackingStores(); + assert.deepEqual(reloaded.get(alpha, "shared"), { v: 1 }); + + assert.equal(await cache.delete(alpha, "shared"), true); + await cache.put(beta, "shared", { v: 2 }); + const retyped = open(); + await retyped.pullAllBackingStores(); + assert.equal(retyped.get(alpha, "shared"), undefined); + assert.deepEqual(retyped.get(beta, "shared"), { v: 2 }); +}); + +test("a put by a type renamed under the stored id, which lists that id compatible, restamps the record; an unrelated type is refused", async () => { + const old = defineDocumentType({ type: "ren.old", schema: z.object({ v: z.number() }), schemaId: "ren.id.v1", schemaName: "ren.old" }); + const renamed = defineDocumentType({ + type: "ren.new", + schema: z.object({ v: z.number() }), + schemaId: "ren.id.v2", + schemaName: "ren.new", + compatibleSchemaIds: ["ren.id.v2", "ren.id.v1"], + }); + const other = defineDocumentType({ type: "ren.other", schema: z.object({ v: z.number() }), schemaId: "ren.id.other", schemaName: "ren.other" }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "cache.msgpack"); + const open = (...types: (typeof old)[]) => + types + .reduce((builder, type) => builder.withDocumentType(type), CultCache.builder()) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + + await open(old).put(old, "k", { v: 1 }); + const cache = open(renamed, other as unknown as typeof old); + await cache.pullAllBackingStores(); + assert.deepEqual(cache.get(renamed, "k"), { v: 1 }); + await assert.rejects(() => cache.put(other, "k", { v: 9 }), SchemaConflictError); + await cache.put(renamed, "k", { v: 2 }); + + const reloaded = open(renamed); + await reloaded.pullAllBackingStores(); + assert.deepEqual(reloaded.get(renamed, "k"), { v: 2 }); + assert.equal(readSingleFileStore(storePath).envelopes.filter((envelope) => envelope.key === "k").length, 1); +}); + test("CultCache can register name and index lookups after entries already exist", async () => { const noteDocument = defineDocumentType({ type: "note", @@ -2582,6 +2639,25 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", } }); +// One key is held once: a push whose type does not own the stored schema id is refused, so the writer cannot emit a store its own +// reader refuses; one whose type owns it (its own id, or one it lists compatible, as C# WrittenOwns) restamps the record in place. +test("SingleFileMessagePackBackingStore push asks whether the written type owns the stored id, not what the types are called", async () => { + const seed = CultCache.builder().withDocumentType(ownsId).withDocumentType(listsId).build(); + await seed.put(ownsId, "k", { name: "owner" }); + await seed.put(listsId, "k", { name: "lister" }); + const file = join(await mkdtemp(join(tmpdir(), "cultcache-push-key-")), "store.msgpack"); + const store = new SingleFileMessagePackBackingStore(file); + const idsOf = async () => ((decode(await readFile(file)) as unknown[][])[2] as unknown[][]).map((record) => [record[0], record[1]]); + + await store.push(seed.getRequiredEnvelope(listsId, "k")); + await assert.rejects(() => store.push(seed.getRequiredEnvelope(ownsId, "k")), SchemaConflictError); + assert.deepEqual(await idsOf(), [["k", "id.lister"]]); + + await store.pushAll([seed.getRequiredEnvelope(listsId, "k")]); + await store.push(seed.getRequiredEnvelope(listsId, "k")); + assert.equal((await store.pullAll()).length, 1); +}); + // Owner beats lister: a definition that owns an id and one that lists it as compatible register together in either order. A record // under the id is the owner's, and a lister's only when nothing owns it; two listers and no owner name no single definition. const ownsId = defineDocumentType({ type: "tests.owns", schema: z.object({ name: z.string() }), schemaId: "id.owned", schemaName: "tests.owns" }); diff --git a/packages/cultcache-ts/test/store-file.test.ts b/packages/cultcache-ts/test/store-file.test.ts index 39ac553f..eb41b77b 100644 --- a/packages/cultcache-ts/test/store-file.test.ts +++ b/packages/cultcache-ts/test/store-file.test.ts @@ -12,6 +12,7 @@ import { decode, encode } from "@msgpack/msgpack"; import { documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; +import { SchemaConflictError, StoreUnreadableError } from "../src/store-format"; import type { CultCacheEnvelope } from "../src/types"; import { @@ -571,15 +572,84 @@ const snapshotOf = async (store: SingleFileMessagePackBackingStore): Promise `${entry.type}/${entry.key}:${[...documentPayloadBytes(entry.payload)].at(-1)}`) .sort(); -test("push replaces the record at the same type and key and keeps every other", async () => { +test("push replaces the record under the same key and type and keeps every other; retyping is delete then push", async () => { const directory = await scratch(); try { const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc")); - await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]); + await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("c", "u", 3)]); await store.push(entryOf("a", "t", 9)); - assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/a:3"]); - await store.push(entryOf("c", "t", 4)); - assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/c:4", "u/a:3"]); + assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/c:3"]); + await store.push(entryOf("d", "t", 4)); + assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/d:4", "u/c:3"]); + await store.delete(entryOf("a", "t")); + await store.push(entryOf("a", "u", 5)); + assert.deepEqual(await snapshotOf(store), ["t/b:2", "t/d:4", "u/a:5", "u/c:3"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a push that would land a second type under an existing key is refused and the file is byte for byte unchanged", async () => { + const directory = await scratch(); + try { + const path = join(directory, "store.cc"); + const store = new SingleFileMessagePackBackingStore(path); + await store.pushAll([entryOf("k", "t", 1), entryOf("m", "t", 2)]); + const before = [...(await readFile(path))]; + await assert.rejects( + () => store.push(entryOf("k", "u", 9)), + (error: unknown) => error instanceof SchemaConflictError && error.recordKey === "k" && /holds record "k" as type "t"/u.test(error.message), + ); + assert.deepEqual([...(await readFile(path))], before); + assert.deepEqual(await snapshotOf(new SingleFileMessagePackBackingStore(path)), ["t/k:1", "t/m:2"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a pushAll batch that carries one key under two types is refused and writes nothing", async () => { + const directory = await scratch(); + try { + const path = join(directory, "store.cc"); + const store = new SingleFileMessagePackBackingStore(path); + await assert.rejects(() => store.pushAll([entryOf("k", "t", 1), entryOf("k", "u", 2)]), SchemaConflictError); + assert.equal((await readdir(directory)).includes("store.cc"), false); + await store.pushAll([entryOf("a", "t", 1)]); + const before = [...(await readFile(path))]; + await assert.rejects(() => store.pushAll([entryOf("k", "t", 1), entryOf("k", "u", 2)]), SchemaConflictError); + assert.deepEqual([...(await readFile(path))], before); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a pushAll batch that carries one key twice under one type keeps the last and the file reads back with one record", async () => { + const directory = await scratch(); + try { + const path = join(directory, "store.cc"); + await new SingleFileMessagePackBackingStore(path).pushAll([entryOf("k", "t", 1), entryOf("a", "t", 3), entryOf("k", "t", 2)]); + assert.deepEqual(await snapshotOf(new SingleFileMessagePackBackingStore(path)), ["t/a:3", "t/k:2"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("a legacy envelope array that holds a key under two types is refused on open, as a v1 store is", async () => { + const directory = await scratch(); + try { + const path = join(directory, "legacy.cc"); + const bytes = encode([ + { key: "a", type: "t", payload: { n: 1 }, storedAt: "2026-10-20T00:00:00Z" }, + { key: "a", type: "u", payload: { n: 2 }, storedAt: "2026-10-20T00:00:00Z" }, + ]); + await writeFile(path, bytes); + const store = new SingleFileMessagePackBackingStore(path); + await assert.rejects( + () => store.pullAll(), + (error: unknown) => error instanceof StoreUnreadableError && /"a" appears more than once/u.test(error.message), + ); + await assert.rejects(() => store.push(entryOf("b")), StoreUnreadableError); + assert.deepEqual([...(await readFile(path))], [...bytes]); } finally { await rm(directory, { recursive: true, force: true }); } @@ -589,11 +659,12 @@ test("delete removes only the record at that type and key", async () => { const directory = await scratch(); try { const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc")); - await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]); + await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("c", "u", 3)]); await store.delete(entryOf("a", "t")); - assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]); + assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/c:3"]); await store.delete(entryOf("zzz", "t")); - assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]); + await store.delete(entryOf("c", "t")); + assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/c:3"]); } finally { await rm(directory, { recursive: true, force: true }); } diff --git a/packages/cultcache-ts/test/unique-index.test.ts b/packages/cultcache-ts/test/unique-index.test.ts index 67af97fb..d36c2cf2 100644 --- a/packages/cultcache-ts/test/unique-index.test.ts +++ b/packages/cultcache-ts/test/unique-index.test.ts @@ -119,14 +119,6 @@ test("a removed or changed holder releases its value", async () => { assert.equal(cache.getKeyByIndex(document, "sku", "T"), "a"); }); -test("the keys in a refusal are in ordinal order whichever one lands second", async () => { - const document = item("item"); - const cache = memory(document); - await cache.put(document, "m", { sku: "S" }); - await assert.rejects(cache.put(document, "z", { sku: "S" }), { message: refusal("m", "z", "tests.item", "sku") }); - await assert.rejects(cache.put(document, "B", { sku: "S" }), { message: refusal("B", "m", "tests.item", "sku") }); -}); - // The C# White_Space set, and nothing else, is blank. Every UTF-16 code unit is judged, so a rule written for a few // characters (or for JavaScript's \s, which counts U+FEFF and omits U+0085) cannot pass. const CSHARP_WHITE_SPACE = (code: number) => @@ -365,6 +357,77 @@ test("vectors duplicate-store.cc and blank-store.cc hold their verdicts", async } }); +// refusals.msgpack: rows [label, ops, refusal] written by the C# reference. TypeScript has no batch commit +// (variants:idea:unique-index-runtimes.ts-batch-commit), so it performs every row whose commits each hold one record: +// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 of the README (20). +const REFUSAL_SCHEMAS: [string, string, string, Record][] = [ + ["unique_item", "tests.unique_item", "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", { Sku: "sku" }], + ["uq_pair", "tests.uq_pair", "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", { Sku: "sku", Alt: "second" }], + ["uq_alias", "tests.uq_alias", "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", { Sku: "sku", alt: "second" }], + ["uq_wide", "tests.uq_wide", "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", { "\uFF21": "sku", "\u{10400}": "second" }], + ["uq_alpha", "tests.uq_alpha", "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", { Sku: "sku" }], + ["uq_beta", "tests.uq_beta", "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", { Sku: "sku" }], +]; + +type RefusalOp = ["open", string | null] | ["pull", string] | ["commit", [string, string, string, string][]]; + +test("vectors refusals.msgpack: every row TS can perform", async () => { + const rows = decode(await readFile(join(vectors, "refusals.msgpack"))) as [string, RefusalOp[], [string, string, string, string] | null][]; + const definitions = new Map( + REFUSAL_SCHEMAS.map(([type, schemaName, schemaId, indexes]) => [ + schemaName, + defineDocumentType({ + type, + schemaId, + schemaName, + schemaVersion: `${schemaName}.v1`, + schema: z.object({ sku: z.string(), second: z.string() }), + codec: codecs.slots<{ sku: string; second: string }>({ 0: ["sku", codecs.string], 1: ["second", codecs.string] }, `Uq${type}`), + indexes, + }), + ]), + ); + const dir = await mkdtemp(join(tmpdir(), "uidx-refusals-")); + let walked = 0; + try { + for (const [label, ops, refused] of rows) { + if (ops.some((op) => op[0] === "commit" && op[1].length > 1)) continue; + const file = join(dir, `${walked}.cc`); + let cache: CultCache | undefined; + const perform = async (op: RefusalOp) => { + if (op[0] === "open") { + if (op[1] !== null) await copyFile(join(vectors, op[1]), file); + let builder = CultCache.builder().withGenericStore(new SingleFileMessagePackBackingStore(file)); + for (const definition of definitions.values()) builder = builder.withDocumentType(definition); + cache = builder.build(); + await cache.pullAllBackingStores(); + } else if (op[0] === "pull") { + await copyFile(join(vectors, op[1]), file); + await cache!.pullAllBackingStores(); + } else { + for (const [key, schema, sku, second] of op[1]) await cache!.put(definitions.get(schema)!, key, { sku, second }); + } + }; + for (const op of ops.slice(0, -1)) await perform(op); + const last = ops[ops.length - 1]; + if (refused === null) { + await perform(last); + } else { + const [k1, k2, schema, alias] = refused; + const before = last[0] === "commit" || (last[0] === "open" && last[1] === null) + ? (existsSync(file) ? await readFile(file) : Buffer.alloc(0)) + : await readFile(join(vectors, last[1]!)); + await assert.rejects(perform(last), { message: refusal(k1, k2, schema, alias) }, label); + assert.deepEqual(existsSync(file) ? await readFile(file) : Buffer.alloc(0), before, `${label}: the file is untouched`); + } + walked++; + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + assert.equal(walked, 20); +}); + test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => { const dir = await mkdtemp(join(tmpdir(), "uidx-load-")); try { @@ -381,11 +444,21 @@ test("a refused load leaves the cache's prior state as it was, and a later good } }); -test("registerIndex over held duplicates installs nothing", async () => { +test("registerIndex refuses a name the definition declares as a role", async () => { const document = item("item", "roles"); const cache = memory(document); await cache.put(document, "a", { sku: "S", group: "g" }); await cache.put(document, "b", { sku: "S", group: "h" }); + await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: /declared as a role; a role is never a unique index/u }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined); + await cache.put(document, "c", { sku: "S", group: "i" }); +}); + +test("registerIndex over held duplicates installs nothing", async () => { + const document = item("item", "none"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S", group: "g" }); + await cache.put(document, "b", { sku: "S", group: "h" }); await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: refusal("a", "b", "tests.item", "sku") }); assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined, "no map was installed"); await cache.put(document, "c", { sku: "S", group: "i" }); @@ -468,7 +541,7 @@ test("a load judges every unique alias: a store collision on the second alias re }); test("registerIndex of a second unique alias judges it, naming the two smallest keys whatever order they were written in", async () => { - const document = pair({ indexes: { a: "a" }, roles: { b: "b" } }); + const document = pair({ indexes: { a: "a" } }); const cache = memory(document); await cache.put(document, "z", { a: "1", b: "s" }); await cache.put(document, "m", { a: "2", b: "s" }); @@ -478,27 +551,33 @@ test("registerIndex of a second unique alias judges it, naming the two smallest assert.equal(cache.getKeyByIndex(document, "b", "s"), undefined); }); -test("verdict parity: the same stores are refused whatever the load order, and the first collision refuses (text is informational)", async () => { - const dir = await mkdtemp(join(tmpdir(), "uidx-verdict-")); - try { - // Three records, one pair colliding; whichever position the pair has in the file, the store is refused and a clean store opens. - const roles = pair({ roles: { a: "a" } }); - for (const [name, values, refused] of [ - ["pair-first", ["s", "s", "t"], true], - ["pair-last", ["t", "s", "s"], true], - ["pair-split", ["s", "t", "s"], true], - ["distinct", ["r", "s", "t"], false], - ] as const) { - const file = join(dir, `${name}.cc`); - const writer = CultCache.builder().withDocumentType(roles).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); - for (const [i, a] of values.entries()) await writer.put(roles, `r${i}`, { a }); - const unique = pair({ indexes: { a: "a" } }); - const reader = CultCache.builder().withDocumentType(unique).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); - if (refused) await assert.rejects(reader.pullAllBackingStores(), /both hold unique index 'a'/u, name); - else await reader.pullAllBackingStores(); - assert.equal(reader.snapshot().length, refused ? 0 : 3, name); - } - } finally { - await rm(dir, { recursive: true, force: true }); - } +// ---- the single-put judge's two orders, which no pair-order vector row reaches from TS (a vector row's second record +// arrives in a batch that TS cannot perform): aliases by UTF-16 code unit, then the key pair by UTF-16 code unit ---- + +test("a put colliding on two aliases names the one least by UTF-16 code unit, not by code point", async () => { + // U+10400 is the surrogate pair D801 DC00, so it sorts before U+FF21 by code unit and after it by code point. + const document = pair({ indexes: { "Ａ": "a", "\u{10400}": "b" } }); + const cache = memory(document); + await cache.put(document, "h1", { a: "s1", b: "t1" }); + await cache.put(document, "h2", { a: "s2", b: "t2" }); + await assert.rejects(cache.put(document, "p", { a: "s1", b: "t2" }), { message: refusal("h2", "p", "tests.pair", "\u{10400}") }); + assert.equal(cache.get(document, "p"), undefined); +}); + +test("a put names its key pair by UTF-16 code unit, not by normalised form", async () => { + // "e" U+0302 is the NFD form of U+00EA: equal when normalised, and less by code unit. + const document = item("item"); + const cache = memory(document); + await cache.put(document, "ê", { sku: "v" }); + await assert.rejects(cache.put(document, "eÌ‚", { sku: "v" }), { message: refusal("eÌ‚", "ê", "tests.item", "sku") }); + assert.equal(cache.get(document, "eÌ‚"), undefined); +}); + +test("a put colliding on two aliases orders them by UTF-16 code unit, not by normalised form", async () => { + // U+00EA normalises to e U+0302, which is less than e U+0303 when normalised and greater by code unit. + const document = pair({ indexes: { "ê": "a", "ẽ": "b" } }); + const cache = memory(document); + await cache.put(document, "h1", { a: "s1", b: "t1" }); + await cache.put(document, "h2", { a: "s2", b: "t2" }); + await assert.rejects(cache.put(document, "p", { a: "s1", b: "t2" }), { message: refusal("h2", "p", "tests.pair", "ẽ") }); }); diff --git a/packages/cultmath/src/CultMath/CultMath.csproj b/packages/cultmath/src/CultMath/CultMath.csproj index ed99c004..d3846d4f 100644 --- a/packages/cultmath/src/CultMath/CultMath.csproj +++ b/packages/cultmath/src/CultMath/CultMath.csproj @@ -10,7 +10,7 @@ GameCult.Math MIT AND MPL-2.0 AND Apache-2.0 - 0.5.0 + 0.5.1 Clean-room C# math shaped for HLSL-style renderer code. GameCult README.md diff --git a/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md b/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md index 7edfa1b2..ebe21e7d 100644 --- a/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md +++ b/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this package are documented in this file. Earlier releases (0.2.0-0.2.3) predate this package's changelog and are not backfilled here; see `docs/semver-policy.md` for why. -## [Unreleased] +## [0.5.1] - 2026-11-11 ### Added diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll index 205d3f9b..7757c254 100644 Binary files a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll and b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll differ diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb index 9f865c09..e0991c71 100644 Binary files a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb and b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb differ diff --git a/packages/cultmath/unity/org.gamecult.cultmath/package.json b/packages/cultmath/unity/org.gamecult.cultmath/package.json index 20ffe196..e06e6c89 100644 --- a/packages/cultmath/unity/org.gamecult.cultmath/package.json +++ b/packages/cultmath/unity/org.gamecult.cultmath/package.json @@ -1,7 +1,7 @@ { "name": "org.gamecult.cultmath", "displayName": "CultMath", - "version": "0.5.0", + "version": "0.5.1", "unity": "2021.3", "description": "Precompiled CultMath numeric primitives and matching HLSL semantics for GameCult Unity runtimes.", "license": "MIT AND MPL-2.0 AND Apache-2.0", diff --git a/packages/cultnet-py/src/cultnet_py/schema_catalog.py b/packages/cultnet-py/src/cultnet_py/schema_catalog.py index 85ab799d..b640b753 100644 --- a/packages/cultnet-py/src/cultnet_py/schema_catalog.py +++ b/packages/cultnet-py/src/cultnet_py/schema_catalog.py @@ -429,9 +429,9 @@ def wire_message_schema_properties(schema_version: str) -> dict[str, Any]: "messageId": common["messageId"], "subscriptionId": common["subscriptionId"], "changeKind": {"enum": ["added", "updated", "removed"]}, - "document": {"$ref": "#/$defs/rawDocumentRecord"}, - "schemaId": common["schemaId"], - "recordKey": common["recordKey"], + "document": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/rawDocumentRecord"}]}, + "schemaId": {"type": ["string", "null"], "minLength": 1}, + "recordKey": {"type": ["string", "null"], "minLength": 1}, "commitId": {"type": ["string", "null"], "minLength": 1}, "commitCount": {"type": ["integer", "null"], "minimum": 2}, }, diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 0c8bcbf3..68f8f8ea 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -2315,9 +2315,10 @@ class CultNetTests(unittest.TestCase): change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"]) self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"]) - self.assertIn("document", change_schema["properties"]) - self.assertIn("schemaId", change_schema["properties"]) - self.assertIn("recordKey", change_schema["properties"]) + # The C# writer sets every key, nil where unset (a removed change's document, an added change's recordKey and schemaId). + self.assertEqual(change_schema["properties"]["document"]["oneOf"][0], {"type": "null"}) + self.assertEqual(change_schema["properties"]["schemaId"], {"type": ["string", "null"], "minLength": 1}) + self.assertEqual(change_schema["properties"]["recordKey"], {"type": ["string", "null"], "minLength": 1}) self.assertEqual(change_schema["properties"]["commitId"], {"type": ["string", "null"], "minLength": 1}) self.assertEqual(change_schema["properties"]["commitCount"], {"type": ["integer", "null"], "minimum": 2}) diff --git a/packages/cultnet-rs/tests/database_change_commit_vector.rs b/packages/cultnet-rs/tests/database_change_commit_vector.rs new file mode 100644 index 00000000..322227dc --- /dev/null +++ b/packages/cultnet-rs/tests/database_change_commit_vector.rs @@ -0,0 +1,43 @@ +//! `tests/vectors/database-change-commit/two-change-commit.msgpack` is written only by the C# reference: two +//! `cultnet.database_change_raw.v0` messages of one commit, every key present, nil where unset. cultnet-rs has no typed +//! `database_change_raw` message yet, so this walks the stream with the same `rmpv` values its wire decoder builds on and +//! pins the keys and values a Rust adopter will read. A rename of `commitId` or `commitCount` in the C# writer fails here. + +use rmpv::Value; +use std::io::Cursor; + +fn messages() -> Vec { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../tests/vectors/database-change-commit/two-change-commit.msgpack"); + let bytes = std::fs::read(path).expect("the vector is committed"); + let mut cursor = Cursor::new(bytes.as_slice()); + let mut out = Vec::new(); + while (cursor.position() as usize) < bytes.len() { + out.push(rmpv::decode::read_value(&mut cursor).expect("a msgpack value")); + } + out +} + +fn field<'a>(message: &'a Value, key: &str) -> &'a Value { + message + .as_map() + .and_then(|entries| entries.iter().find(|(k, _)| k.as_str() == Some(key))) + .map(|(_, v)| v) + .unwrap_or_else(|| panic!("the C# writer wrote no key {key}")) +} + +#[test] +fn a_two_change_commit_written_by_csharp_carries_its_commit_id_count_and_changes() { + let messages = messages(); + assert_eq!(messages.len(), 2); + for message in &messages { + assert_eq!(field(message, "schemaVersion").as_str(), Some("cultnet.database_change_raw.v0")); + assert_eq!(field(message, "commitId").as_str(), Some("9f1c0f0e4b7a4c1f9d2e")); + assert_eq!(field(message, "commitCount").as_u64(), Some(2)); + assert_eq!(field(message, "changeKind").as_str(), Some("removed")); + assert_eq!(field(message, "subscriptionId").as_str(), Some("sub-1")); + assert_eq!(field(message, "schemaId").as_str(), Some("schema-note")); + assert!(field(message, "document").is_nil(), "a removed change's document is nil"); + } + let keys: Vec<_> = messages.iter().map(|m| field(m, "recordKey").as_str().unwrap().to_string()).collect(); + assert_eq!(keys, ["note:a", "note:b"]); +} diff --git a/packages/cultnet-ts/test/database-change-commit-vector.test.ts b/packages/cultnet-ts/test/database-change-commit-vector.test.ts new file mode 100644 index 00000000..fea121c0 --- /dev/null +++ b/packages/cultnet-ts/test/database-change-commit-vector.test.ts @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import Ajv2020 from "ajv/dist/2020.js"; +import { decodeMulti } from "@msgpack/msgpack"; + +import { cultNetSchemas, parseCultNetMessage } from "../src/index"; + +// tests/vectors/database-change-commit/two-change-commit.msgpack is written only by the C# reference: two removed +// database_change_raw messages of one commit, every key present, nil where unset (document included). +const vector = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "database-change-commit", "two-change-commit.msgpack"); +const changes = () => [...decodeMulti(readFileSync(vector))] as Record[]; + +test("a two-change commit written by C# decodes with its commitId, commitCount and changes", () => { + const messages = changes().map((message) => parseCultNetMessage(structuredClone(message))); + assert.equal(messages.length, 2); + for (const message of messages) { + assert.equal(message.schemaVersion, "cultnet.database_change_raw.v0"); + assert.equal((message as { commitId?: unknown }).commitId, "9f1c0f0e4b7a4c1f9d2e"); + assert.equal((message as { commitCount?: unknown }).commitCount, 2); + assert.equal((message as { changeKind?: unknown }).changeKind, "removed"); + assert.equal((message as { subscriptionId?: unknown }).subscriptionId, "sub-1"); + assert.equal((message as { schemaId?: unknown }).schemaId, "schema-note"); + } + assert.deepEqual(messages.map((message) => (message as { recordKey?: unknown }).recordKey), ["note:a", "note:b"]); +}); + +test("the database_change_raw schema accepts the C# vector's removed changes (document: nil) and still refuses an added change without a document", () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }); + ajv.addSchema(cultNetSchemas.rawDocumentRecordSchema); + const validate = ajv.compile(cultNetSchemas.databaseChangeRawSchema); + for (const change of changes()) { + assert.equal(change.document, null, "the C# writer wrote the key, nil"); + assert.equal(validate(change), true, JSON.stringify(validate.errors)); + assert.equal(validate({ ...change, changeKind: "added" }), false, "an added change needs a document"); + } +}); diff --git a/packages/cultnet-ts/test/rudp-server.test.ts b/packages/cultnet-ts/test/rudp-server.test.ts index 6ba976d3..0f913055 100644 --- a/packages/cultnet-ts/test/rudp-server.test.ts +++ b/packages/cultnet-ts/test/rudp-server.test.ts @@ -115,6 +115,8 @@ test("a client reconnecting from the same endpoint replaces only its own session assert.throws(() => server.send(oldPeer, "schema", enc("stale")), /not connected/); assert.equal(server.peers.length, 2); + // The server accepting is not the client hearing it: send only once the client's new session is connected. + await waitFor(() => a.connection.connected, "the reconnecting client's handshake"); a.connection.send("schema", enc("after")); b.connection.send("schema", enc("still")); await waitFor(() => a.received.includes("re:after") && b.received.includes("re:still"), "replies after the reconnect"); diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 55fd7584..0acee656 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -310,7 +310,7 @@ public static class CultDocumentMessagePackSerialization /// /// Reads a store file: exactly one complete MessagePack array of three slots (header, schema catalog, records), or an empty /// array, which is an empty store. A truncated file, bytes after the array, a value that is not an array, a first slot that - /// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, or a body + /// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, two records under one key, or a body /// that does not decode are refused with /// , so a reader never takes part of a file for the whole and never skips a slot /// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced. @@ -366,6 +366,7 @@ public static class CultDocumentMessagePackSerialization } RequirePublishedSchemas(snapshot); + RequireUniqueKeys(snapshot); return (snapshot, catalogBytes, recordBytes); } catch (CultStoreUnreadableException) @@ -397,6 +398,17 @@ public static class CultDocumentMessagePackSerialization } } + // A key names one record: identity is the key string alone, compared by ordinal code units (no case folding, no normalisation). + private static void RequireUniqueKeys(CultPersistedStoreSnapshot snapshot) + { + var seen = new HashSet(StringComparer.Ordinal); + foreach (var record in snapshot.Records) + { + if (!seen.Add(record.Key)) + throw new CultStoreUnreadableException($"Record '{record.Key}' appears more than once."); + } + } + // Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is // judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read. private static void JudgeStamp(byte[] payload, Action? requireReadable) diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index d34490f7..5e7156b0 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -59,6 +59,10 @@ await cache.PullAllBackingStoresAsync(); - The raw CultNet document lane should treat these payload bytes as already blessed, not decode and re-encode them for sport. +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` (a directory store `v5`); the single-file and directory stores here write the mark from content. See the Store format and rollback section of the GameCult.Caching README. Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. The only carry-back is narrow and lossy (a whole-store write that leaves no id in the store drops the mark and the ids); otherwise restore from a copy taken before upgrading, and upgrade every reader before any writer. + ## MessagePack source generator NuGet consumers get `[assembly: MessagePackAssumedFormattable(typeof(CultRecordRef<>))]` from the package, so a diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 709fc64d..6f04d99c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3495,10 +3495,11 @@ namespace GameCult.Caching } } - // The one index rule, judged on the commit's result whatever the write order: within one document type and one - // [CultIndex] alias, no two records hold one non-blank value. The holders of a value are the held key (unless this - // admission replaces or evicts it) plus every landing key projecting it; two distinct holders refuse, naming both - // keys and the index and never the value. One pass over what lands, one lookup per (index, value). + // The one index rule, a function of the state the admission would leave and never of staging, file or pull order: + // within one document type and one [CultIndex] alias, no two records hold one non-blank value. The holders of a + // value are the held key (unless this admission replaces or evicts it) plus every landing key projecting it. Over + // every (type, alias, value) held at two or more distinct keys, the refusal names the least by (alias, its least + // key, its second-least key, schema name), each compared by UTF-16 code unit, and never names the value. private void RefuseIndexSharing(VariantPlan plan, IReadOnlyList evicted) { var landing = plan.Admitted.Concat(plan.Dependents).ToList(); @@ -3520,17 +3521,34 @@ namespace GameCult.Caching } } - foreach (var ((index, value), landedKeys) in landed.OrderBy(pair => pair.Key.Index.Alias, StringComparer.Ordinal)) + (string Alias, string First, string Second, string Schema)? least = null; + foreach (var ((index, value), landedKeys) in landed) { var holders = landedKeys.AsEnumerable(); if (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKey) && !replaced.Contains(heldKey)) holders = holders.Append(heldKey); var distinct = holders.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).Take(2).ToArray(); - if (distinct.Length > 1) - throw new InvalidOperationException( - $"Records {distinct[0]} and {distinct[1]} ({schemaNames[index.Type]}) " + - $"both hold unique index '{index.Alias}'; an index value names one record."); + if (distinct.Length < 2) + continue; + var candidate = (index.Alias, distinct[0], distinct[1], schemaNames[index.Type]); + if (least == null || CompareRefusals(candidate, least.Value) < 0) + least = candidate; } + + if (least is { } pair) + throw new InvalidOperationException( + $"Records {pair.First} and {pair.Second} ({pair.Schema}) " + + $"both hold unique index '{pair.Alias}'; an index value names one record."); + } + + private static int CompareRefusals( + (string Alias, string First, string Second, string Schema) left, + (string Alias, string First, string Second, string Schema) right) + { + var order = StringComparer.Ordinal.Compare(left.Alias, right.Alias); + if (order == 0) order = StringComparer.Ordinal.Compare(left.First, right.First); + if (order == 0) order = StringComparer.Ordinal.Compare(left.Second, right.Second); + return order != 0 ? order : StringComparer.Ordinal.Compare(left.Schema, right.Schema); } private static InvalidOperationException CycleRefusal(IEnumerable keys) => diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 38dfaaae..97a183e9 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -399,7 +399,7 @@ namespace GameCult.Caching continue; if (shape.Derive == null && !IsRandomId(id!)) throw new CultElementIdException( - $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex; for readable ids, derive them from a member with [CultElementId(nameof(Member))].", + $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex. To keep an existing readable id, leave that member as it is and add a trailing keyed string member marked [CultElementId(nameof(Member))], where Member is the existing one; the element id is derived from it once and then persisted.", rootPath, path, id); if (!taken.Add(id!)) throw Duplicate(id!, path); diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index 471361dd..6defed61 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -228,6 +228,60 @@ the text of that member (a string or number on the same type, unique within its list), and a derived id may be any text. A readable name set by hand on a non-derived list id is refused; the same member outside a list is unconstrained. +### A type that already has a readable id + +If the element type already has a readable string id (`Name`, `Slug`, a catalog +key), leave it as it is. Add a trailing keyed string member marked +`[CultElementId(nameof(ExistingId))]`; the element id is derived from the +existing member on load or first write, and then persisted: + +```csharp +[MessagePackObject] +public sealed class Recipe +{ + [Key(0)] public string Name = string.Empty; // existing readable id + [Key(1)] public int Yield; + [Key(2)] [CultElementId(nameof(Name))] public string ElementId = string.Empty; // new, last +} +``` + +Marking `Name` itself `[CultElementId]` is refused (`Name` is not 12 lowercase +hex), and that is deliberate: + +- Minted and hand-written ids share one grammar, 12 lowercase hex, so they can + never collide; a derived id is the one readable form and is checked for + uniqueness within its list. +- An id the consumer can edit is not a stable override address. Variants address + an element by its id, so a field that is renamed in place would silently move + the overrides to a different element, or to none. +- A derived id is persisted once. Renaming `Name` later does not change + `ElementId`, so the element and every override that addresses it stay put. + `ElementId` is the address; `Name` goes back to being ordinary data. + +Adopting the recipe on data that already exists: the existing member becomes an +id source, so it must be non-empty and unique within its list. A record with a +duplicate value and an empty one both refuse to load +(`CultElementIdException`), so clean such data first. + +Take the next free `[Key]` slot, after the highest key on the type (and above +the subclasses' keys on a union base). + +## Store format and rollback + +A store that holds an element id is written as `cultcache.store.v3` (a directory +store as `cultcache.store.v5.directory-content-addressed-pages`, `v4` otherwise). +A single-file store that holds a variant but no id is `v2`, and a store with +neither stays `v1`. The mark follows content: a whole-store write that leaves no +element holding an id drops it. + +Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. + +Rolling a consumer back after it has written a v3 or v5 store is therefore +one-way. The only carry-back is narrow and lossy: a whole-store write that leaves +no id anywhere in the store (every record rewritten without list elements) writes +the store unmarked again, and the ids are gone. Otherwise restore from a copy +taken before upgrading. Upgrade every reader before any writer. + ## Persistence discipline Persistence stays manual by default. Mutations mark the cache and attached diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 2bc1d9df..a54251ed 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -142,7 +142,7 @@ namespace GameCult.Caching.Tests public void TheHexRefusalNamesTheDerivedFormForReadableIds() { var refusal = Assert.Throws(() => Assign(new HHasIds { Items = { new EIded { Id = "root" } } }, "r", deterministic: false))!; - Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]")); + Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]").And.Contain("trailing keyed string member").And.Contain("existing readable id")); } [Test] diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 1b6b3fe2..a6c68c94 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -389,6 +389,44 @@ namespace GameCult.Caching.Tests Assert.That(refusal.Message, Does.Contain("'7'")); } + // The documented recipe for a type that already has a readable id (src/GameCult.Caching/README.md, Element ids). + [Test] + public void ATrailingMemberDerivedFromAnExistingReadableIdIsPersistedOnceAndTheStoreIsMarkedV3() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RecipeBook) }); + var path = PathOf("recipes.cc"); + var key = new CultRecordKey("book"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook { Recipes = { new IdRecipe { Name = "bread" }, new IdRecipe { Name = "soup" } } }, key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + var book = cache.Get(key)!; + Assert.That(book.Recipes.Select(recipe => recipe.ElementId), Is.EqualTo(new[] { "bread", "soup" })); + book.Recipes[0].Name = "loaf"; + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), book, key)); + } + + using var again = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + var reread = again.Get(key)!; + Assert.That(reread.Recipes.Select(recipe => recipe.Name), Is.EqualTo(new[] { "loaf", "soup" })); + Assert.That(reread.Recipes.Select(recipe => recipe.ElementId), Is.EqualTo(new[] { "bread", "soup" }), "renaming the source does not move the element"); + } + + [Test] + public void AWholeStoreWriteThatLeavesNoIdInTheStoreDropsTheMarkBackToV1() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RecipeBook) }); + var path = PathOf("drop.cc"); + var key = new CultRecordKey("book"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook { Recipes = { new IdRecipe { Name = "bread" } } }, key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook(), key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the mark follows content: no element holds an id, so the store is unmarked"); + } + // ---- C2a fix batch: the marker, refusals at load, override values, atomic minting ---- private const string HexA = "aaaaaaaaaaaa"; @@ -1359,6 +1397,20 @@ namespace GameCult.Caching.Tests [Key(1)] public List Labels { get; set; } = new(); } + [CultDocument("tests.element_id_recipe_book", "tests.element_id_recipe_book.v1")] + [MessagePackObject] + public sealed class RecipeBook + { + [Key(0)] public List Recipes { get; set; } = new(); + } + + [MessagePackObject] + public sealed class IdRecipe + { + [Key(0)] public string Name { get; set; } = ""; + [Key(1)] [CultElementId(nameof(Name))] public string ElementId { get; set; } = ""; + } + [MessagePackObject] public sealed class IdLabel { diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs index 4aef72a1..3ee64e07 100644 --- a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs +++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs @@ -74,6 +74,18 @@ namespace GameCult.Caching.Tests return path; } + [Test] + public void ADuplicateKeyIsRefusedNamingTheKeyAndTheFile() + { + var path = Path.Combine(_directory, "dup.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "dup-key-same-schema-v3.bin"))); + + Assert.That(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }), + Throws.TypeOf() + .With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path) + .And.Message.Contains("more than once")); + } + [TestCaseSource(nameof(Vectors))] public void OpenReadsExactlyTheFilesTheVectorsSayItReads(string vector, bool reads) { diff --git a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs index d7c68459..1fd026ae 100644 --- a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs +++ b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs @@ -290,5 +290,359 @@ namespace GameCult.Caching.Tests } } } + + // ---- refusals.msgpack: [label, ops, refusal]; the pair a refusal names is a function of the resulting state ---- + + private sealed record Schema(string Name, Type Role, Type Index, string SecondField); + + private static Schema Plain(string suffix, string name) => new( + name, + EmittedDocumentTypes.Emit("UqVec" + suffix + "Role", name, name + ".v1", RoleFields), + EmittedDocumentTypes.Emit("UqVec" + suffix + "Index", name, name + ".v1", IndexFields), + "Category"); + + private static readonly Schema[] Schemas = + { + new("tests.unique_item", ItemRole, ItemIndex, "Category"), + new("tests.uq_pair", + EmittedDocumentTypes.Emit("UqVecPairRole", "tests.uq_pair", "tests.uq_pair.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "Alt") }), + EmittedDocumentTypes.Emit("UqVecPairIndex", "tests.uq_pair", "tests.uq_pair.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "Alt") }), + "Alt"), + new("tests.uq_alias", + EmittedDocumentTypes.Emit("UqVecAliasRole", "tests.uq_alias", "tests.uq_alias.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "alt") }), + EmittedDocumentTypes.Emit("UqVecAliasIndex", "tests.uq_alias", "tests.uq_alias.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "alt") }), + "Alt"), + new("tests.uq_wide", + EmittedDocumentTypes.Emit("UqVecWideRole", "tests.uq_wide", "tests.uq_wide.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "\uFF21"), new("Alt", typeof(string), 1, Role: "\U00010400") }), + EmittedDocumentTypes.Emit("UqVecWideIndex", "tests.uq_wide", "tests.uq_wide.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "\uFF21"), new("Alt", typeof(string), 1, Index: "\U00010400") }), + "Alt"), + Plain("Alpha", "tests.uq_alpha"), + Plain("Beta", "tests.uq_beta"), + }; + + private static Schema SchemaNamed(string name) => Schemas.Single(schema => schema.Name == name); + + private sealed record Rec(string Key, string Schema, string Sku, string Second = ""); + + private sealed record Op(string Kind, string? File = null, Rec[]? Records = null); + + private sealed record Refusal(string K1, string K2, string Schema, string Alias); + + private sealed record Case(string Label, Op[] Ops, Refusal? Refusal); + + private const string Item = "tests.unique_item"; + private const string Pair = "tests.uq_pair"; + private const string Alias = "tests.uq_alias"; + private const string Wide = "tests.uq_wide"; + private const string Alpha = "tests.uq_alpha"; + private const string Beta = "tests.uq_beta"; + + private static Rec[] Items_(params (string Key, string Sku)[] rows) => rows.Select(row => new Rec(row.Key, Item, row.Sku)).ToArray(); + private static Op Open(string? file = null) => new("open", file); + private static Op Pull(string file) => new("pull", file); + private static Op Commit(params Rec[] records) => new("commit", Records: records); + + private static readonly Rec[] PairRecords = { new("item:a", Pair, "S", "X"), new("item:b", Pair, "S", "Y"), new("item:c", Pair, "T", "Y") }; + private static readonly Rec[] SurrogateRecords = Items_(("k\uFF01", "S"), ("k\U0001F600", "S"), ("k\uE000", "S")); + private static readonly Rec[] TypeRecords = + { + new("b:1", Alpha, "P"), new("a:1", Beta, "P"), new("b:2", Alpha, "P"), new("a:2", Beta, "P") + }; + private static readonly Rec[] AliasRecords = { new("item:a", Alias, "S", "X"), new("item:b", Alias, "S", "Y"), new("item:c", Alias, "T", "Y") }; + private static readonly Rec[] RepullBefore = Items_(("item:a1", "A"), ("item:b1", "C")); + private static readonly Rec[] RepullAfter = Items_(("item:a1", "A"), ("item:b1", "B"), ("item:b2", "B"), ("item:c1", "A")); + private static readonly Rec[] KeyCaseRecords = Items_(("a", "S"), ("z", "S"), ("B", "T"), ("C", "T")); + private static readonly Rec[] HolderCaseRecords = Items_(("a", "S"), ("A", "S")); + private static readonly Rec[] CrossSchemaRecords = { new("x:1", Alpha, "P"), new("y:1", Beta, "P") }; + // Distinct under any trim, case-fold, NFC or NFKC of the value: 'S' and 's', 'S' and 'S ', NFC and NFD, fullwidth S. + private static readonly Rec[] DistinctValueRecords = Items_( + ("k:1", "S"), ("k:2", "s"), ("k:3", "S "), ("k:4", "\u00E9"), ("k:5", "e\u0301"), ("k:6", "\uFF33"), ("k:7", " S")); + private static readonly Rec[] NormalKeyRecords = Items_(("\u00E9", "S"), ("e\u0301", "S")); + // Three holders of one value: code-unit order is e+U+0302, f, U+00E9; NFC would order f, U+00E9, e+U+0302 and NFD U+00E9, e+U+0302, f. + private static readonly Rec[] ThreeHolderRecords = Items_(("e\u0302", "S"), ("f", "S"), ("\u00E9", "S")); + // Both records violate both aliases: by code unit U+10400 (D801 DC00) precedes U+FF21, by code point U+FF21 does. + private static readonly Rec[] WideRecords = { new("item:a", Wide, "S", "X"), new("item:b", Wide, "S", "X") }; + private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T")); + private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T")); + + private static readonly (string File, Rec[] Records)[] RefusalStores = + { + ("pair-store.cc", PairRecords), ("surrogate-store.cc", SurrogateRecords), ("types-store.cc", TypeRecords), + ("repull-before.cc", RepullBefore), ("repull-after.cc", RepullAfter), ("alias-store.cc", AliasRecords), + ("key-case-store.cc", KeyCaseRecords), ("key-punctuation-store.cc", KeyPunctuationRecords), + ("surrogate-groups-store.cc", SurrogateGroupRecords), + ("holder-case-store.cc", HolderCaseRecords), + ("cross-schema-store.cc", CrossSchemaRecords), ("distinct-values-store.cc", DistinctValueRecords), + ("normal-keys-store.cc", NormalKeyRecords), ("normal-three-store.cc", ThreeHolderRecords), ("wide-store.cc", WideRecords), + }; + + private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku"); + private static readonly Refusal TypesRefusal = new("a:1", "a:2", Beta, "Sku"); + private static readonly Refusal PairRefusal = new("item:b", "item:c", Pair, "Alt"); + private static readonly Refusal AliasRefusal = new("item:a", "item:b", Alias, "Sku"); + private static readonly Refusal RepullRefusal = new("item:a1", "item:c1", Item, "Sku"); + private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku"); + private static readonly Refusal HolderCaseRefusal = new("A", "a", Item, "Sku"); + private static readonly Refusal NormalKeyRefusal = new("e\u0301", "\u00E9", Item, "Sku"); + private static readonly Refusal ThreeHolderRefusal = new("e\u0302", "f", Item, "Sku"); + private static readonly Refusal WideRefusal = new("item:a", "item:b", Wide, "\U00010400"); + private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku"); + private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku"); + + private static readonly Case[] Cases = + { + new("batch-later-value-staged-first", new[] { Open(), Commit(Items_(("x", "A"), ("y", "A"), ("a", "B"), ("b", "B"))) }, + new Refusal("a", "b", Item, "Sku")), + new("batch-surrogate-keys", new[] { Open(), Commit(SurrogateRecords) }, SurrogateRefusal), + new("batch-interleaved-types", new[] { Open(), Commit(TypeRecords) }, TypesRefusal), + new("batch-two-aliases", new[] { Open(), Commit(PairRecords) }, PairRefusal), + new("held-then-landing", new[] { Open(), Commit(Items_(("m", "S"))), Commit(Items_(("a", "S"))) }, + new Refusal("a", "m", Item, "Sku")), + new("held-in-the-least-group", new[] { Open(), Commit(Items_(("m", "A"))), Commit(Items_(("z", "B"), ("y", "B"), ("a", "A"))) }, + new Refusal("a", "m", Item, "Sku")), + new("swap-off-a-held-value", new[] { Open(), Commit(Items_(("a1", "A"))), Commit(Items_(("a1", "B"), ("c1", "A"))) }, null), + new("load-two-aliases", new[] { Open("pair-store.cc") }, PairRefusal), + new("load-surrogate-keys", new[] { Open("surrogate-store.cc") }, SurrogateRefusal), + new("load-interleaved-types", new[] { Open("types-store.cc") }, TypesRefusal), + new("repull-held-pair", new[] { Open("repull-before.cc"), Pull("repull-after.cc") }, RepullRefusal), + new("load-repull-after", new[] { Open("repull-after.cc") }, RepullRefusal), + new("commit-onto-a-loaded-holder", new[] { Open("repull-before.cc"), Commit(Items_(("item:z", "A"))) }, + new Refusal("item:a1", "item:z", Item, "Sku")), + // Rows 14-19 extend the spec's floor of 13: orderings a code-point, case-insensitive or culture compare would flip. + new("batch-key-case", new[] { Open(), Commit(KeyCaseRecords) }, KeyCaseRefusal), + new("batch-key-punctuation", new[] { Open(), Commit(KeyPunctuationRecords) }, KeyPunctuationRefusal), + new("batch-key-case-in-a-group", new[] { Open(), Commit(Items_(("a", "S"), ("B", "S"))) }, + new Refusal("B", "a", Item, "Sku")), + new("batch-surrogate-groups", new[] { Open(), Commit(SurrogateGroupRecords) }, SurrogateGroupRefusal), + new("batch-alias-case", new[] { Open(), Commit(AliasRecords) }, AliasRefusal), + new("load-alias-case", new[] { Open("alias-store.cc") }, AliasRefusal), + // Rows 20-22 load rows 14, 15 and 17's records, so a runtime with no batch commit still walks cross-group key order. + new("load-key-case", new[] { Open("key-case-store.cc") }, KeyCaseRefusal), + new("load-key-punctuation", new[] { Open("key-punctuation-store.cc") }, KeyPunctuationRefusal), + new("load-surrogate-groups", new[] { Open("surrogate-groups-store.cc") }, SurrogateGroupRefusal), + // Rows 23-24: two holders of one value that differ only by case are two holders; a case-folding dedupe would see one. + new("batch-holders-by-case", new[] { Open(), Commit(HolderCaseRecords) }, HolderCaseRefusal), + new("load-holders-by-case", new[] { Open("holder-case-store.cc") }, HolderCaseRefusal), + // Row 25: the held `a` is replaced only by the exact key `a`, never by `A`. + new("batch-replaced-by-case", new[] { Open(), Commit(Items_(("a", "S"))), Commit(Items_(("A", "S"))) }, HolderCaseRefusal), + // Rows 26-31: identity is exact. One value in two schemas is two groups; values differing in case, trailing or leading space, normalisation form or width are two values; keys differing only by normalisation form are two holders. + new("batch-value-in-two-schemas", new[] { Open(), Commit(CrossSchemaRecords) }, null), + new("load-value-in-two-schemas", new[] { Open("cross-schema-store.cc") }, null), + new("batch-distinct-values", new[] { Open(), Commit(DistinctValueRecords) }, null), + new("load-distinct-values", new[] { Open("distinct-values-store.cc") }, null), + new("batch-keys-by-normal-form", new[] { Open(), Commit(NormalKeyRecords) }, NormalKeyRefusal), + new("load-keys-by-normal-form", new[] { Open("normal-keys-store.cc") }, NormalKeyRefusal), + // Rows 32-33: one landing record sharing two unique aliases with a held one; the least alias by code unit is named ('Alt' before 'Sku'; 'Sku' before 'alt'). + new("put-onto-a-held-pair", new[] { Open(), Commit(new Rec("m", Pair, "S", "X")), Commit(new Rec("a", Pair, "S", "X")) }, + new Refusal("a", "m", Pair, "Alt")), + new("put-onto-a-held-alias-pair", new[] { Open(), Commit(new Rec("m", Alias, "S", "X")), Commit(new Rec("a", Alias, "S", "X")) }, + new Refusal("a", "m", Alias, "Sku")), + // Rows 34-35: three holders of one value; the two least by code unit are named, which no NFC, NFD or culture sort of the keys yields. + new("batch-three-holders-normal-form", new[] { Open(), Commit(ThreeHolderRecords) }, ThreeHolderRefusal), + new("load-three-holders-normal-form", new[] { Open("normal-three-store.cc") }, ThreeHolderRefusal), + // Rows 36-37: two records violating two aliases, U+FF21 and U+10400; aliases compare by code unit, so U+10400 is named. + new("batch-alias-code-unit", new[] { Open(), Commit(WideRecords) }, WideRefusal), + new("load-alias-code-unit", new[] { Open("wide-store.cc") }, WideRefusal), + }; + + private static byte[] RefusalStoreBytes(Rec[] records) + { + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxr-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + var path = Path.Combine(directory, "store.cc"); + var roles = records.Select(record => SchemaNamed(record.Schema).Role).Distinct().ToArray(); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions + { + Registry = CultDocumentRegistry.ForTypes(roles), StoreFlushOnDispose = true + })) + { + cache.Clock = () => StoredAt; + foreach (var record in records) + { + var schema = SchemaNamed(record.Schema); + cache.UpsertAsync(schema.Role, EmittedDocumentTypes.New(schema.Role, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key)) + .GetAwaiter().GetResult(); + } + + cache.FlushAsync().GetAwaiter().GetResult(); + } + + return File.ReadAllBytes(path); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + private static byte[] RefusalsFile() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(Cases.Length); + foreach (var row in Cases) + { + writer.WriteArrayHeader(3); + writer.Write(row.Label); + writer.WriteArrayHeader(row.Ops.Length); + foreach (var op in row.Ops) + { + writer.WriteArrayHeader(2); + writer.Write(op.Kind); + if (op.Records != null) + { + writer.WriteArrayHeader(op.Records.Length); + foreach (var record in op.Records) + { + writer.WriteArrayHeader(4); + writer.Write(record.Key); + writer.Write(record.Schema); + writer.Write(record.Sku); + writer.Write(record.Second); + } + } + else if (op.File == null) writer.WriteNil(); + else writer.Write(op.File); + } + + if (row.Refusal == null) writer.WriteNil(); + else + { + writer.WriteArrayHeader(4); + writer.Write(row.Refusal.K1); + writer.Write(row.Refusal.K2); + writer.Write(row.Refusal.Schema); + writer.Write(row.Refusal.Alias); + } + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + // Performs one op of a row against the open cache; the cache's own file is the only state it leaves behind. + private static void Perform(ref CultCache? cache, string path, Op op) + { + var indexTypes = Schemas.Select(schema => schema.Index).ToArray(); + switch (op.Kind) + { + case "open": + if (op.File != null) File.WriteAllBytes(path, Committed(op.File)); + cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(indexTypes) }); + break; + case "pull": + File.WriteAllBytes(path, Committed(op.File!)); + cache!.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + break; + case "commit": + cache!.Commit(batch => + { + foreach (var record in op.Records!) + { + var schema = SchemaNamed(record.Schema); + batch.Upsert(schema.Index, EmittedDocumentTypes.New(schema.Index, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key)); + } + }); + break; + default: + throw new InvalidOperationException(op.Kind); + } + } + + [Test] + public void RefusalsHoldTheRule() + { + Check("refusals.msgpack", RefusalsFile()); + foreach (var (file, records) in RefusalStores) + Check(file, RefusalStoreBytes(records)); + + // The committed file is what is walked, not the in-memory table. + var reader = new MessagePackReader(Committed("refusals.msgpack")); + var count = reader.ReadArrayHeader(); + Assert.That(count, Is.EqualTo(Cases.Length)); + var walked = 0; + var failed = new List(); + for (var i = 0; i < count; i++) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(3)); + var label = reader.ReadString()!; + var ops = new List(); + for (var n = reader.ReadArrayHeader(); n > 0; n--) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2)); + var kind = reader.ReadString()!; + if (kind == "commit") + { + var records = new List(); + for (var m = reader.ReadArrayHeader(); m > 0; m--) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4)); + records.Add(new Rec(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!)); + } + + ops.Add(new Op(kind, Records: records.ToArray())); + } + else ops.Add(new Op(kind, NextText(ref reader))); + } + + Refusal? refusal = null; + if (!reader.TryReadNil()) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4)); + refusal = new Refusal(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!); + } + + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxf-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + CultCache? cache = null; + try + { + var path = Path.Combine(directory, "store.cc"); + foreach (var op in ops.Take(ops.Count - 1)) + Perform(ref cache, path, op); + + var last = ops[^1]; + if (refusal == null) + { + Assert.DoesNotThrow(() => Perform(ref cache, path, last), label); + } + else + { + cache?.FlushAsync().GetAwaiter().GetResult(); + var before = last.Kind == "open" && last.File != null ? Committed(last.File) + : last.Kind == "pull" ? Committed(last.File!) + : File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty(); + CultCache? attempt = cache; + var refused = Assert.Throws(() => Perform(ref attempt, path, last), label)!; + Assert.That(refused.Message, Is.EqualTo( + $"Records {refusal.K1} and {refusal.K2} ({refusal.Schema}) both hold unique index '{refusal.Alias}'; an index value names one record."), label); + if (last.Kind != "open") cache?.FlushAsync().GetAwaiter().GetResult(); + Assert.That(File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty(), Is.EqualTo(before), $"{label}: the file is untouched"); + } + + walked++; + } + catch (AssertionException failure) + { + failed.Add($"{label}: {failure.Message}"); + } + finally + { + cache?.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + + Assert.That(failed, Is.Empty, "rows that failed"); + Assert.That(walked, Is.EqualTo(37)); + } } } diff --git a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs index 4172c83b..ec9dde5b 100644 --- a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs @@ -1,5 +1,7 @@ #nullable enable using System; +using System.Buffers; +using System.Collections.Generic; using System.IO; using System.Text.Json; using System.Threading.Tasks; @@ -246,6 +248,39 @@ namespace GameCult.Networking.Tests Assert.That(Convert.ToHexString(CultNetSchemaMessageSerialization.Serialize(message)), Is.EqualTo("85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D93E6D6573736167655F696E76616C69643A2052617720707574206D65737361676573206D75737420636F6E7461696E206120646F63756D656E74206D61702EAB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A8646F63756D656E74A576616C7565C0A461734F66C0A763757272656E74C0")); } + // The C# writer sets every key (nil where unset), so a removed change carries document: nil, and an added change + // carries recordKey/schemaId: nil. The schema describes those bytes; it still refuses an added change with no document. + private static List CommitVectorChanges() + { + var path = Path.Combine(RepoRoot(), "tests", "vectors", "database-change-commit", "two-change-commit.msgpack"); + var reader = new MessagePackReader(File.ReadAllBytes(path)); + var changes = new List(); + while (!reader.End) + changes.Add(JsonDocument.Parse(MessagePackSerializer.ConvertToJson(reader.ReadRaw().ToArray())).RootElement); + return changes; + } + + [Test] + public void DatabaseChangeRaw_TheCommitVectorsRemovedChangesWithANilDocument_MatchTheSchema() + { + var validator = new MiniJsonSchemaValidator(SchemaDir()); + var changes = CommitVectorChanges(); + Assert.That(changes, Has.Count.EqualTo(2)); + foreach (var change in changes) + { + Assert.That(change.GetProperty("document").ValueKind, Is.EqualTo(JsonValueKind.Null), "the C# writer wrote the key, nil"); + validator.AssertValid(change, "cultnet.database-change-raw.schema.json"); + } + } + + [Test] + public void DatabaseChangeRaw_AnAddedChangeWithANilDocument_IsRefused() + { + var validator = new MiniJsonSchemaValidator(SchemaDir()); + var removed = CommitVectorChanges()[0].GetRawText().Replace("\"removed\"", "\"added\""); + Assert.Throws(() => validator.AssertValid(JsonDocument.Parse(removed).RootElement, "cultnet.database-change-raw.schema.json")); + } + private static void AssertErrorMatchesSchema(CultNetErrorMessage message) { var validator = new MiniJsonSchemaValidator(SchemaDir()); diff --git a/tests/vectors/document-variants-c2a/readability/distinct-keys-case-v3.bin b/tests/vectors/document-variants-c2a/readability/distinct-keys-case-v3.bin new file mode 100644 index 00000000..031d29fb --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/distinct-keys-case-v3.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item“”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta”¥Alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/distinct-keys-nfd-v3.bin b/tests/vectors/document-variants-c2a/readability/distinct-keys-nfd-v3.bin new file mode 100644 index 00000000..ba97c3cd --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/distinct-keys-nfd-v3.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item’”¢Ã©¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”£e̬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/dup-key-different-payload-v3.bin b/tests/vectors/document-variants-c2a/readability/dup-key-different-payload-v3.bin new file mode 100644 index 00000000..ac4744fc --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/dup-key-different-payload-v3.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item“”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/dup-key-other-schema-v3.bin b/tests/vectors/document-variants-c2a/readability/dup-key-other-schema-v3.bin new file mode 100644 index 00000000..b28cb1d7 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/dup-key-other-schema-v3.bin @@ -0,0 +1 @@ +“²cultcache.store.v3’—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item—­vectors.other­vectors.other°vectors.other.v1­vectors.otherÙN{"schemaName":"vectors.other","schemaVersion":"vectors.other.v1","members":[]}‘­vectors.other“”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¥alpha­vectors.other¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/dup-key-same-schema-v3.bin b/tests/vectors/document-variants-c2a/readability/dup-key-same-schema-v3.bin new file mode 100644 index 00000000..250b432e --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/dup-key-same-schema-v3.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item“”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/generate.py b/tests/vectors/document-variants-c2a/readability/generate.py index 25e12496..fad89a24 100644 --- a/tests/vectors/document-variants-c2a/readability/generate.py +++ b/tests/vectors/document-variants-c2a/readability/generate.py @@ -5,6 +5,7 @@ A file may be replaced by a flush or a commit exactly when the runtime's own rea covers open, flush and commit. Each vector is the bytes named; the trailing and truncated ones are cut from v3-base. """ import pathlib +import unicodedata import msgpack @@ -12,12 +13,21 @@ here = pathlib.Path(__file__).resolve().parent whole = (here.parent / "v3-base.msgpack").read_bytes() pack = lambda value: msgpack.packb(value, use_bin_type=True) base = msgpack.unpackb(whole, raw=False) +# A second published schema, and a record of it under the key alpha. +other_entry = ["vectors.other", "vectors.other", "vectors.other.v1", "vectors.other", '{"schemaName":"vectors.other","schemaVersion":"vectors.other.v1","members":[]}', ["vectors.other"], []] +other_record = ["alpha", "vectors.other"] + base[2][0][2:] +# The same visible key as U+00E9 (NFC) and U+0065 U+0301 (NFD): equal after normalisation, unequal ordinally. +nfc, nfd = "\u00e9", "e\u0301" +assert unicodedata.normalize("NFC", nfd) == unicodedata.normalize("NFC", nfc) and nfc != nfd +assert "Alpha".casefold() == "alpha" and "Alpha" != "alpha" vectors = { "zero-byte": b"", "empty-array": pack([]), # A legacy envelope array: an array of maps. Rust, TypeScript and Python read it; C# has no legacy reader. "legacy-envelopes": pack([{"key": "old", "type": "old.type", "payload": pack([]), "storedAt": "2026-01-01T00:00:00Z"}]), + # A legacy envelope array holding key a under two types: a key is one record, whatever the type. Refused like a v3 store with a shared key. + "legacy-dup-key": pack([{"key": "a", "type": "old.one", "payload": pack([]), "storedAt": "2026-01-01T00:00:00Z"}, {"key": "a", "type": "old.two", "payload": pack([]), "storedAt": "2026-01-01T00:00:00Z"}]), "legacy-trailing": None, # filled below: the legacy array plus bytes after it "ints": pack([1, 2, 3]), "nil-first": pack([None, "x"]), @@ -55,6 +65,16 @@ vectors = { "variant-slot-v3": pack([base[0], base[1], base[2] + [["gamma"] + base[2][0][1:3] + [b"", ["alpha", [[0, [[0, ""]], "", "gamma"]]]]]]), # v3-base with a fourth top-level slot after the records. "extra-top-slot": pack(base + [0]), + # A store holding two records under one key is unreadable, whatever else they agree or differ on. Every record names a + # schema the catalog publishes, so the shared key is the only defect. + "dup-key-same-schema-v3": pack([base[0], base[1], base[2] + [base[2][0]]]), + # Two records under one key, each naming a different published schema: the key alone is the identity. + "dup-key-other-schema-v3": pack([base[0], base[1] + [other_entry], [base[2][0], other_record, base[2][1]]]), + # alpha twice with different payload bytes: last-wins would silently pick one. + "dup-key-different-payload-v3": pack([base[0], base[1], base[2] + [base[2][0][:3] + [b"\x92\xa5alpha\x09"]]]), + # Keys are ordinal strings: no case folding and no Unicode normalisation, so these are distinct keys and the store reads. + "distinct-keys-case-v3": pack([base[0], base[1], base[2] + [["Alpha"] + base[2][0][1:]]]), + "distinct-keys-nfd-v3": pack([base[0], base[1], [[nfc] + base[2][0][1:], [nfd] + base[2][1][1:]]]), } vectors["legacy-trailing"] = vectors["legacy-envelopes"] + b"\x01\x02\x03" for name, data in vectors.items(): diff --git a/tests/vectors/document-variants-c2a/readability/legacy-dup-key.bin b/tests/vectors/document-variants-c2a/readability/legacy-dup-key.bin new file mode 100644 index 00000000..c2b117cc --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/legacy-dup-key.bin @@ -0,0 +1 @@ +’„£key¡a¤type§old.one§payloadĨstoredAt´2026-01-01T00:00:00Z„£key¡a¤type§old.two§payloadĨstoredAt´2026-01-01T00:00:00Z \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/manifest.txt b/tests/vectors/document-variants-c2a/readability/manifest.txt index dd1fc4b4..6d7af19d 100644 --- a/tests/vectors/document-variants-c2a/readability/manifest.txt +++ b/tests/vectors/document-variants-c2a/readability/manifest.txt @@ -1,9 +1,11 @@ # Whether each runtime's own reader opens the file. A flush and a commit replace a file exactly when it reads; where a runtime has a soft flush (TypeScript, Rust) it leaves a file that reads byte for byte and refuses one that does not. +# A store holding two records under one key is unreadable; keys are ordinal strings (no case folding, no normalisation), so distinct spellings are distinct keys. # Columns: file, csharp, rust, typescript, python. `../v3-base.msgpack` is the one valid store. ../v3-base.msgpack reads reads reads reads zero-byte.bin refuses refuses refuses refuses empty-array.bin reads reads reads reads legacy-envelopes.bin refuses reads reads reads +legacy-dup-key.bin refuses refuses refuses refuses legacy-trailing.bin refuses refuses refuses refuses ints.bin refuses refuses refuses refuses nil-first.bin refuses refuses refuses refuses @@ -30,3 +32,9 @@ missing-schema-versioned-v3.bin refuses refuses refuses refuses compatible-id-only-v3.bin reads reads reads reads own-id-over-compatible-v3.bin reads reads reads reads compatible-before-owner-v3.bin reads reads reads reads +dup-key-same-schema-v3.bin refuses refuses refuses refuses +dup-key-other-schema-v3.bin refuses refuses refuses refuses +dup-key-different-payload-v3.bin refuses refuses refuses refuses +distinct-keys-case-v3.bin reads reads reads reads +distinct-keys-nfd-v3.bin reads reads reads reads +../../document-variants-c0/current-catalog-bad-canonical.msgpack refuses refuses refuses refuses diff --git a/tests/vectors/unique-index/README.md b/tests/vectors/unique-index/README.md index 548c8845..55b393f3 100644 --- a/tests/vectors/unique-index/README.md +++ b/tests/vectors/unique-index/README.md @@ -18,7 +18,66 @@ reviewed commit. Other runtimes are held to the same canonical text and the same stored-at pinned to `2026-01-01T00:00:00Z`. Records `item:a` and `item:b` both hold Sku `S-1`; `item:c` holds Sku `" "` and `item:d` holds Sku `U+0085`. Written through a role Sku. - `blank-store.cc`: the same, only `item:c` and `item:d`. +- `pair-store.cc`, `surrogate-store.cc`, `types-store.cc`, `repull-before.cc`, `repull-after.cc`, `alias-store.cc`, + `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`, `cross-schema-store.cc`, `distinct-values-store.cc`, `normal-keys-store.cc`, `normal-three-store.cc`, `wide-store.cc`: single-file stores written like + `duplicate-store.cc`, through roles, for the rows of `refusals.msgpack` that open or pull them. Schemas + `tests.unique_item`, `tests.uq_pair` (Sku and Alt both unique), `tests.uq_alias` (Sku and `alt` both unique), + `tests.uq_wide` (Sku and Alt both unique, aliases U+FF21 and U+10400), `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus + `.v1`. The schema ids C# writes in the store catalogs and on every record: + - `tests.unique_item`: `sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882` + - `tests.uq_pair`: `sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60` + - `tests.uq_alias`: `sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f` + - `tests.uq_wide`: `sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f` + - `tests.uq_alpha`: `sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389` + - `tests.uq_beta`: `sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586` - `outcomes.msgpack`: rows `[file, unique alias or nil, open|refused, k1 or nil, k2 or nil]`. Opening `file` with that alias a unique index (nil: a role, no unique alias) yields the verdict. A refusal reads, byte for byte, `Records and (tests.unique_item) both hold unique index ''; an index value names one record.` with `k1 < k2` ordinal, never contains the value, and leaves the file byte-identical. + +## refusals.msgpack + +RULE: a unique-index refusal is a function of the state the admission would leave, never of staging, file or pull +order. Over every (type, unique alias, non-blank value) that state holds at two or more distinct keys, name the one +least by (alias, its least key, its second-least key, schema name), each compared by UTF-16 code unit (C# +`StringComparer.Ordinal`): `Records and () both hold unique index ''; an index value names one +record.` with `k1 < k2` its two least keys. One rule for a single write, a batch, a load, a re-pull and an index +registration. + +Rows `[label, ops, refusal]`. Each op is `["open", file-or-nil]` (copy that vector file to a fresh path, or nil for a +fresh path with no file, and open a cache on it), `["pull", file]` (overwrite the open cache's file with that vector +file, then pull every backing store again) or `["commit", [[key, schema, sku, second], ...]]` (one batch, staged in row +order; schema is the schema name). `refusal` is nil (every op succeeds) or `[k1, k2, schema, alias]`: every op but the +last succeeds and the last refuses with exactly the RULE text built from those fields, leaving the file +byte-identical. Schemas are those of the store files above; `second` is `Category` (plain) except in `tests.uq_pair` +and `tests.uq_alias`, where it is `Alt` (unique, alias `Alt` and `alt` respectively). + +The 13 rows of the spec are a floor; rows 14-37 extend it with orderings that a code-point, case-insensitive or +culture compare would flip. Rows, in file order: 1 batch-later-value-staged-first, 2 batch-surrogate-keys, 3 +batch-interleaved-types, 4 batch-two-aliases, 5 held-then-landing, 6 held-in-the-least-group, 7 swap-off-a-held-value, +8 load-two-aliases, 9 load-surrogate-keys, 10 load-interleaved-types, 11 repull-held-pair, 12 load-repull-after, 13 +commit-onto-a-loaded-holder, 14 batch-key-case (`B` before `a`), 15 batch-key-punctuation (`Z` before `_` before `a`), +16 batch-key-case-in-a-group, 17 batch-surrogate-groups (U+1F600 before U+E000, the reverse of code-point order), 18 +batch-alias-case (alias `Sku` before `alt`), 19 load-alias-case, 20 load-key-case, 21 load-key-punctuation, 22 +load-surrogate-groups, 23 batch-holders-by-case (`a` and `A` holding one value: `A` before `a`), 24 load-holders-by-case, 25 batch-replaced-by-case (held `a`, then a commit of `A` on its value: `A` does not replace `a`), +26 batch-value-in-two-schemas (one value in `tests.uq_alpha` and `tests.uq_beta` at two keys: two groups, success), 27 +load-value-in-two-schemas, 28 batch-distinct-values (`S`, `s`, `S `, ` S`, U+00E9, `e`+U+0301, U+FF33 are seven values: success), 29 +load-distinct-values, 30 batch-keys-by-normal-form (keys U+00E9 and `e`+U+0301 holding one value are two holders, `e`+U+0301 first), +31 load-keys-by-normal-form, 32 put-onto-a-held-pair (held `m` and landing `a` share Sku and Alt in `tests.uq_pair`, one record per +commit: `Alt` is named before `Sku`), 33 put-onto-a-held-alias-pair (the same in `tests.uq_alias`: `Sku` before `alt`), 34 batch-three-holders-normal-form +(keys `e`+U+0302, `f` and U+00E9 holding one value: the two least by code unit, `e`+U+0302 and `f`, are named; an NFC, NFD or culture sort of the keys names another pair), 35 +load-three-holders-normal-form, 36 batch-alias-code-unit (two records each holding the same value under both aliases of `tests.uq_wide`, U+FF21 and U+10400: by UTF-16 code +unit U+10400 (D801 DC00) precedes U+FF21, by code point it follows it, so the refusal names alias U+10400), 37 load-alias-code-unit. +Rows 19-22, 24, 27, 29, 31, 35 and 37 open a store holding the records of rows 18, 14, 15, 17, 23, 26, 28, 30, 34 and 36, so a runtime with no batch +commit still walks alias and cross-group key order. + +A runtime performs every row its API can perform and asserts the count it walks. These are the counts a runtime must +assert, not a statement of which runtimes walk them today: + +- C# and Rust: all 37. +- Python: 35, every row but 3 and 26 (`put_envelopes` takes one type, so no batch of two types). +- TypeScript: 20, the rows with no multi-record commit: 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 + (no batch commit, so no row 1-4, 6, 7, 14-18, 23, 26, 28, 30, 34 or 36; `variants:idea:unique-index-runtimes.ts-batch-commit`). + +Not pinned: the schema-name tiebreak. Record keys are unique across a whole cache, so two types never hold the same key and +two groups can never tie on alias and key pair; the final `schema` comparison cannot decide a refusal. diff --git a/tests/vectors/unique-index/alias-store.cc b/tests/vectors/unique-index/alias-store.cc new file mode 100644 index 00000000..3ea51ef5 Binary files /dev/null and b/tests/vectors/unique-index/alias-store.cc differ diff --git a/tests/vectors/unique-index/cross-schema-store.cc b/tests/vectors/unique-index/cross-schema-store.cc new file mode 100644 index 00000000..1dd2fb66 Binary files /dev/null and b/tests/vectors/unique-index/cross-schema-store.cc differ diff --git a/tests/vectors/unique-index/distinct-values-store.cc b/tests/vectors/unique-index/distinct-values-store.cc new file mode 100644 index 00000000..6eabdd96 Binary files /dev/null and b/tests/vectors/unique-index/distinct-values-store.cc differ diff --git a/tests/vectors/unique-index/holder-case-store.cc b/tests/vectors/unique-index/holder-case-store.cc new file mode 100644 index 00000000..ed8f9a2a Binary files /dev/null and b/tests/vectors/unique-index/holder-case-store.cc differ diff --git a/tests/vectors/unique-index/key-case-store.cc b/tests/vectors/unique-index/key-case-store.cc new file mode 100644 index 00000000..b5032c8d Binary files /dev/null and b/tests/vectors/unique-index/key-case-store.cc differ diff --git a/tests/vectors/unique-index/key-punctuation-store.cc b/tests/vectors/unique-index/key-punctuation-store.cc new file mode 100644 index 00000000..9f62d0be Binary files /dev/null and b/tests/vectors/unique-index/key-punctuation-store.cc differ diff --git a/tests/vectors/unique-index/normal-keys-store.cc b/tests/vectors/unique-index/normal-keys-store.cc new file mode 100644 index 00000000..27f36b96 Binary files /dev/null and b/tests/vectors/unique-index/normal-keys-store.cc differ diff --git a/tests/vectors/unique-index/normal-three-store.cc b/tests/vectors/unique-index/normal-three-store.cc new file mode 100644 index 00000000..580ae038 Binary files /dev/null and b/tests/vectors/unique-index/normal-three-store.cc differ diff --git a/tests/vectors/unique-index/pair-store.cc b/tests/vectors/unique-index/pair-store.cc new file mode 100644 index 00000000..96a180ff Binary files /dev/null and b/tests/vectors/unique-index/pair-store.cc differ diff --git a/tests/vectors/unique-index/refusals.msgpack b/tests/vectors/unique-index/refusals.msgpack new file mode 100644 index 00000000..01dcc9e1 Binary files /dev/null and b/tests/vectors/unique-index/refusals.msgpack differ diff --git a/tests/vectors/unique-index/repull-after.cc b/tests/vectors/unique-index/repull-after.cc new file mode 100644 index 00000000..3d2a3a13 Binary files /dev/null and b/tests/vectors/unique-index/repull-after.cc differ diff --git a/tests/vectors/unique-index/repull-before.cc b/tests/vectors/unique-index/repull-before.cc new file mode 100644 index 00000000..8ee44ec0 Binary files /dev/null and b/tests/vectors/unique-index/repull-before.cc differ diff --git a/tests/vectors/unique-index/surrogate-groups-store.cc b/tests/vectors/unique-index/surrogate-groups-store.cc new file mode 100644 index 00000000..f9774c85 Binary files /dev/null and b/tests/vectors/unique-index/surrogate-groups-store.cc differ diff --git a/tests/vectors/unique-index/surrogate-store.cc b/tests/vectors/unique-index/surrogate-store.cc new file mode 100644 index 00000000..4d0a1daa Binary files /dev/null and b/tests/vectors/unique-index/surrogate-store.cc differ diff --git a/tests/vectors/unique-index/types-store.cc b/tests/vectors/unique-index/types-store.cc new file mode 100644 index 00000000..36c22d18 Binary files /dev/null and b/tests/vectors/unique-index/types-store.cc differ diff --git a/tests/vectors/unique-index/wide-store.cc b/tests/vectors/unique-index/wide-store.cc new file mode 100644 index 00000000..2dcc1e4a Binary files /dev/null and b/tests/vectors/unique-index/wide-store.cc differ diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index ef98338b..225bb3ae 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -2,15 +2,19 @@ All notable changes to this package are documented in this file. -## [Unreleased] +## Unreleased -Releases as 3.0.0 (major). +### Breaking -### Breaking (major) +- A single-file store holding two records under one key is unreadable: the reader refuses it with `CultStoreUnreadableException` naming the key (keys compare by ordinal code units, no case folding or normalisation), so a cache, `ReadStoreSnapshot` and a flush refuse the file and leave it as it is. Before, the later record silently replaced the earlier. A directory store reads its manifest through the same reader and refuses a duplicate key the same way. Repair the duplicate with the runtime that wrote it before reading the store here. + +## [3.0.0] - 2026-11-11 + +### Breaking - A single-file store flush refuses a file it cannot read (empty, truncated, an unreadable stamp or body) with `CultStoreUnreadableException` and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. - A single-file store reads v1, v2 and v3 stamps and refuses v4 and later by name; a directory store reads v4 and v5 and refuses v6 and later by name. Each judges the stamp before the body, so an unreadable body never makes a newer store look like damage. -- A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. +- A store is written in the format its content needs, and CultLib 2.0 refuses the newer single-file ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. Rolling back to a 2.0 pin after a store has been written as v3 or v5 is one-way: the only carry-back is narrow and lossy (a whole-store write that leaves no element holding an id writes the store unmarked, and the ids are gone), so restore from a copy taken before upgrading. Refusal by name is not uniform in 1.x: a v5 directory store is refused from `cultlib-unity-v1.0.57` (`caching-unity-v1.2.0`), but a v3 single-file store is read silently by every pin before `cultlib-unity-v2.0.0` (including `cultlib-unity-v1.0.61`, `caching-unity-v1.4.0`, `cultlib-v1.0.32`), which drops the ids and can rewrite the elements without them. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). - A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. - A write replaces the record at a key only when the type it writes there owns that record, decided per key and the same for a cache and for a cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll`, in a single-file store and a directory store. The written type owns the stored record when its entry declares the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry and reads it whole (no persisted slot unread): an older id of its own schema name that it reads whole is its own and is restamped, as in 2.0.1. Anything else is refused with `CultSchemaConflictException` and the file left as it was: a record of another registered type, under its current id or an old id it lists, and a record the written type would read only by dropping persisted slots. No registry enters the decision, `CultDocumentRegistry.Shared` included, so a lent registry never changes the answer. This changes a cache: where 2.x let a cache holding two types replace one's record at a key with the other's, it now refuses, with or without a backing store. Retype a key by removing it and adding the new record, in one batch (`batch.Remove(key)` then `batch.Upsert(...)`) or staged apart (`Remove` then `UpsertAsync`); the removal is judged as a removal, so a record no registered type owns is still refused. A removal or check goes to the registry's removal rule, and a cacheless call, which has no registry, removes a record only when an entry it writes owns it (a `Delete` then `PushAll` on an unattached store writes no entry and is refused). diff --git a/unity/org.gamecult.cultlib/README.md b/unity/org.gamecult.cultlib/README.md index e118ac65..30db7fba 100644 --- a/unity/org.gamecult.cultlib/README.md +++ b/unity/org.gamecult.cultlib/README.md @@ -11,19 +11,19 @@ and OpenSSL license notices are included under `Third Party Notices`. Build it with `scripts/build-unity-package.ps1`. Do not edit generated plugin assemblies in the package output. -Install version `2.0.0` through Unity Package Manager with: +Install version `3.0.0` through Unity Package Manager with: ```text -https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v2.0.0 +https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v3.0.0 ``` -The package depends on `org.gamecult.cultmath` (exactly 0.5.0) and does not +The package depends on `org.gamecult.cultmath` (exactly 0.5.1) and does not carry its own `CultMath.dll`. Unity Package Manager resolves git packages only from the project manifest, so add CultMath beside CultLib in `Packages/manifest.json`: ```text -"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath#cultmath-unity-v0.5.0" +"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath#cultmath-unity-v0.5.1" ``` CultMesh chooses mapped memory for reachable same-machine bodies and advertised diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll index bf05fec2..5b040bec 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb index bfe11807..d2629dfd 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll index 8e7b88f1..cff0d028 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb index 3fb019e1..d952c867 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll index 2dd55355..f9674828 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb index 1635f093..f881acde 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll index 6e138fd5..9bff9912 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb index 021de32d..62bdb6b3 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll index ceb2fbf4..f7a86dde 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb index 3074fbd6..d27197b5 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll index 21e64ce1..da269490 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll index dd6482f9..065ac7c5 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb index 4fcf53c6..8cb5dad7 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb differ diff --git a/unity/org.gamecult.cultlib/package.json b/unity/org.gamecult.cultlib/package.json index 19e42b58..1111d3fb 100644 --- a/unity/org.gamecult.cultlib/package.json +++ b/unity/org.gamecult.cultlib/package.json @@ -1,10 +1,10 @@ { "name": "org.gamecult.cultlib", "displayName": "CultLib", - "version": "2.0.1", + "version": "3.0.0", "unity": "2021.3", "dependencies": { - "org.gamecult.cultmath": "0.5.0" + "org.gamecult.cultmath": "0.5.1" }, "description": "CultCache, CultNet, and CultMesh managed runtime assemblies for Unity.", "license": "MIT",