diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index c934c33b..badfadb5 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -234,6 +234,8 @@ class CultCache: @_mutating def register_index(self, document: DocumentDefinition[Any], index: str, extractor: str | Any) -> None: self._assert_registered(document) + if index in document.roles: + raise CultCacheError(f"Index '{index}' of {document.type} is declared as a role; a role is never a unique index.") index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()} index_extractors.setdefault(document.type, {})[index] = extractor self._install_lookups(self._state.name_extractors, index_extractors) @@ -243,7 +245,11 @@ class CultCache: ) -> None: """Derives every lookup under the candidate extractors, then installs extractors and lookups together. An extractor that raises on a held value installs nothing.""" - names, indexes, lookup_keys = self._derive_lookups(self._state.values, name_extractors, index_extractors) + names, indexes, lookup_keys = self._derive_lookups( + [(type, key, value) for type, values in self._state.values.items() for key, value in values.items()], + name_extractors, + index_extractors, + ) ( self._state.name_extractors, self._state.index_extractors, @@ -311,6 +317,7 @@ class CultCache: structures, running every check and every user extractor; the cache's state is replaced only once that build finishes, so a refused load admits nothing.""" loaded_values: dict[str, dict[str, Any]] = {} + read_order: dict[tuple[str, str], Any] = {} # (type, key) -> value, in the order the stores yielded them loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {} loaded_legacy_global_keys: dict[str, str] = {} routes, generic = self._state.stores_by_type, self._state.generic_store @@ -349,9 +356,10 @@ class CultCache: envelope = self._stamped(document, envelope) value = document.decode_payload(envelope.payload) loaded_values.setdefault(envelope.type, {})[envelope.key] = value + read_order[(envelope.type, envelope.key)] = value loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups( - loaded_values, self._state.name_extractors, self._state.index_extractors + [(type, key, value) for (type, key), value in read_order.items()], self._state.name_extractors, self._state.index_extractors ) # Plain attribute assignments: nothing fallible runs once the swap begins. ( @@ -615,24 +623,22 @@ class CultCache: def _derive_lookups( self, - values_by_type: dict[str, dict[str, Any]], + records: list[tuple[str, str, Any]], name_extractors: dict[str, str | Any], index_extractors: dict[str, dict[str, str | Any]], ) -> tuple[_Names, _Indexes, _LookupKeysByType]: - """Builds name and index lookups for the given values in fresh dicts, touching no cache state.""" + """Builds name and index lookups for the given (type, key, value) records, judged in the order given, in fresh dicts, touching no cache state.""" names: _Names = {} indexes: _Indexes = {} lookup_keys: _LookupKeysByType = {} - for type, values in values_by_type.items(): - name_extractor = name_extractors.get(type) - type_index_extractors = index_extractors.get(type, {}) - for key, value in values.items(): - lookup_keys.setdefault(type, {})[key] = self._lookup_keys_for(value, name_extractor, type_index_extractors) + landing: list[tuple[str, str, _LookupKeys]] = [] + for type, key, value in records: + keys = self._lookup_keys_for(value, name_extractors.get(type), index_extractors.get(type, {})) + lookup_keys.setdefault(type, {})[key] = keys + landing.append((type, key, keys)) # Judged whole, over every type at once and on the file alone (never against what the cache held): a held value can # have been mutated in place since it was admitted, and another process may have moved a value between records. - self._refuse_index_sharing( - [(type, key, keys) for type, by_key in lookup_keys.items() for key, keys in by_key.items()], {}, () - ) + self._refuse_index_sharing(landing, {}, ()) for type in lookup_keys: for key, keys in lookup_keys.get(type, {}).items(): if keys.name is not None: diff --git a/packages/cultcache-py/tests/test_unique_index.py b/packages/cultcache-py/tests/test_unique_index.py index 0dca2aa8..2d988e78 100644 --- a/packages/cultcache-py/tests/test_unique_index.py +++ b/packages/cultcache-py/tests/test_unique_index.py @@ -544,6 +544,41 @@ class BatchAndOrderTests(unittest.TestCase): cache.pull_all_backing_stores() self.assertEqual(str(caught.exception), refusal("s:1", "s:2", "second", "alpha")) + def test_aliases_sort_by_utf16_code_unit_not_code_point(self) -> None: + astral, bmp = "k\U0001F600", "k\uff01" # code points: bmp < astral; UTF-16 units: D83D < FF01 + document = item_doc(indexes={bmp: "sku", astral: "serial"}, roles={}) + cache = cache_of(document) + rows = [Item(sku="S", serial="T"), Item(sku="S", serial="T")] + with self.assertRaises(CultCacheError) as caught: + cache.put_envelopes(document, [envelope(document, f"item:{n}", row) for n, row in enumerate(rows)]) + self.assertEqual(str(caught.exception), refusal("item:0", "item:1", "item", astral)) + + def test_a_load_judges_records_in_the_order_the_file_yields_them(self) -> None: + alpha, beta = item_doc("alpha", indexes={"k": "sku"}, roles={}), item_doc("beta", indexes={"k": "sku"}, roles={}) + loose_alpha, loose_beta = item_doc("alpha", indexes={}, roles={}), item_doc("beta", indexes={}, roles={}) + store = ListStore( + [ + envelope(loose_alpha, "a:0", Item(sku="Q")), + envelope(loose_beta, "b:1", Item(sku="S")), + envelope(loose_beta, "b:2", Item(sku="S")), + envelope(loose_alpha, "a:1", Item(sku="P")), + envelope(loose_alpha, "a:2", Item(sku="P")), + ] + ) + cache = cache_of(alpha, beta, store=store) + with self.assertRaises(CultCacheError) as caught: + cache.pull_all_backing_stores() + self.assertEqual(str(caught.exception), refusal("b:1", "b:2", "beta", "k")) + + def test_register_index_refuses_an_alias_the_definition_declares_as_a_role(self) -> None: + document = item_doc() + cache = cache_of(document) + with self.assertRaises(CultCacheError): + cache.register_index(document, "group", "group") + cache.put_envelopes( + document, [envelope(document, "item:1", Item(group="guide")), envelope(document, "item:2", Item(group="guide"))] + ) + class LoadFreshnessTests(unittest.TestCase): def test_a_load_is_judged_on_the_file_alone_not_against_the_held_map(self) -> None: diff --git a/packages/cultmesh-py/tests/test_unique_index_watch.py b/packages/cultmesh-py/tests/test_unique_index_watch.py index 44c606ff..ccc0b44d 100644 --- a/packages/cultmesh-py/tests/test_unique_index_watch.py +++ b/packages/cultmesh-py/tests/test_unique_index_watch.py @@ -36,6 +36,16 @@ class WatchByIndexTests(unittest.TestCase): self.assertEqual(misheard, []) self.assertIsNone(node.database.get_by_index(document, "price", str_text)) + def test_a_watch_hears_a_record_leaving_a_value_by_the_canonical_text(self) -> None: + document = define_database_entry_type("mesh.leaving", [("price", 0)], indexes={"price": "price"}) + node = CultMesh.create_node(runtime_id="mesh-watch-leaving") + node.database.register_document(document) + node.database.put(document, "item:1", {"price": 2.0}) + heard: list[str] = [] + node.database.watch_by_index(document, "price", "2", lambda change: heard.append(change.record_key)) + node.database.put(document, "item:1", {"price": 3.0}) + self.assertEqual(heard, ["item:1"]) + def test_a_blank_value_is_never_indexed_so_never_watched(self) -> None: document = define_database_entry_type("mesh.blank", [("sku", 0)], indexes={"sku": "sku"}) node = CultMesh.create_node(runtime_id="mesh-watch-blank") diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 5ece34be..0c8bcbf3 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -13,6 +13,7 @@ from datetime import UTC, datetime from pathlib import Path from typing import Callable from uuid import uuid4 +import msgpack from cultcache_py import CultCache, define_database_entry_type from cultnet_py.benchmark import run_benchmark from cultnet_py.compare_csharp import ( @@ -1600,25 +1601,13 @@ class CultNetTests(unittest.TestCase): with self.subTest(field=field, bad=bad), self.assertRaisesRegex(ValueError, field): CultNetDatabaseChange.from_wire({**base, field: bad}) - def test_a_two_change_commit_from_the_csharp_server_parses(self) -> None: - # CultNetDatabaseServer.DeliverCommit (CultNetDatabaseServer.cs:791-797) tags every change of a commit with its - # commitId and commitCount = the number of changes delivered, so a swap of two records arrives with commitCount 2. - # Shaped as the C# [MessagePackObject] writes it: every key present, nil where unset. - swap = [ - { - "schemaVersion": "cultnet.database_change_raw.v0", - "messageId": f"change-{index}", - "subscriptionId": "sub-1", - "changeKind": "updated", - "document": None, - "recordKey": key, - "schemaId": "schema-note", - "commitId": "9f1c0f0e4b7a4c1f9d2e", - "commitCount": 2, - } - for index, key in enumerate(("note:a", "note:b")) - ] - changes = [CultNetDatabaseChange.from_wire(message) for message in swap] + def test_a_two_change_commit_written_by_csharp_parses(self) -> None: + # tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, written by the + # C# serializer (CultNetReplicaCommitTests), so a rename or retyping of commitId/commitCount fails here. + vector = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "database-change-commit" / "two-change-commit.msgpack" + messages = list(msgpack.Unpacker(io.BytesIO(vector.read_bytes()), raw=False)) + self.assertEqual(len(messages), 2) + changes = [CultNetDatabaseChange.from_wire(message) for message in messages] self.assertEqual([change.commit_count for change in changes], [2, 2]) self.assertEqual({change.commit_id for change in changes}, {"9f1c0f0e4b7a4c1f9d2e"}) self.assertEqual([change.to_wire()["commitCount"] for change in changes], [2, 2]) diff --git a/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs b/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs index 6059d7ba..b38592ec 100644 --- a/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs @@ -225,6 +225,39 @@ namespace GameCult.Networking.Tests Assert.That(run.Faults, Has.Count.EqualTo(1)); } + [Test] + public void ATwoChangeCommitIsWrittenAsTheVectorOtherRuntimesDecode() + { + // tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, as the C# + // serializer writes them, concatenated. Written with CULTCACHE_WRITE_VECTORS=1; every other run compares bytes. + var commit = Enumerable.Range(0, 2).Select(index => new CultNetDatabaseChangeRawMessage + { + MessageId = $"change-{index}", + SubscriptionId = "sub-1", + ChangeKind = "removed", + RecordKey = index == 0 ? "note:a" : "note:b", + SchemaId = "schema-note", + CommitId = "9f1c0f0e4b7a4c1f9d2e", + CommitCount = 2 + }); + var bytes = commit.SelectMany(message => CultNetSchemaMessageSerialization.Serialize(message)).ToArray(); + + var directory = TestContext.CurrentContext.TestDirectory; + while (directory != null && !System.IO.Directory.Exists(System.IO.Path.Combine(directory, "tests", "vectors"))) + directory = System.IO.Path.GetDirectoryName(directory); + Assert.That(directory, Is.Not.Null, "tests/vectors is above the test directory"); + var path = System.IO.Path.Combine(directory!, "tests", "vectors", "database-change-commit", "two-change-commit.msgpack"); + if (Environment.GetEnvironmentVariable("CULTCACHE_WRITE_VECTORS") == "1") + { + System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!); + System.IO.File.WriteAllBytes(path, bytes); + return; + } + + Assert.That(System.IO.File.Exists(path), Is.True, "the vector is committed"); + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(System.IO.File.ReadAllBytes(path)))); + } + [Test] public void AnOlderPeerReadsTheChangeAndIgnoresTheCommitFields() { diff --git a/tests/vectors/database-change-commit/README.md b/tests/vectors/database-change-commit/README.md new file mode 100644 index 00000000..39a78fa1 --- /dev/null +++ b/tests/vectors/database-change-commit/README.md @@ -0,0 +1,10 @@ +# database-change-commit vectors + +Written only by the C# reference, `ATwoChangeCommitIsWrittenAsTheVectorOtherRuntimesDecode` in +`tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs`, with `CULTCACHE_WRITE_VECTORS=1`. Every other run +compares the bytes. + +- `two-change-commit.msgpack`: two `cultnet.database_change_raw.v0` messages (`removed`, `note:a` and `note:b`) + concatenated, each as the C# `[MessagePackObject]` writes it (every key present, nil where unset), both carrying + `commitId` `9f1c0f0e4b7a4c1f9d2e` and `commitCount` 2. Read it as a msgpack stream. Python walks it in + `packages/cultnet-py/tests/test_cultnet.py`. diff --git a/tests/vectors/database-change-commit/two-change-commit.msgpack b/tests/vectors/database-change-commit/two-change-commit.msgpack new file mode 100644 index 00000000..055dc4ae --- /dev/null +++ b/tests/vectors/database-change-commit/two-change-commit.msgpack @@ -0,0 +1 @@ +‰­schemaVersion¾cultnet.database_change_raw.v0©messageId¨change-0®subscriptionId¥sub-1ªchangeKind§removed¨documentÀ©recordKey¦note:a¨schemaId«schema-note¨commitId´9f1c0f0e4b7a4c1f9d2e«commitCount‰­schemaVersion¾cultnet.database_change_raw.v0©messageId¨change-1®subscriptionId¥sub-1ªchangeKind§removed¨documentÀ©recordKey¦note:b¨schemaId«schema-note¨commitId´9f1c0f0e4b7a4c1f9d2e«commitCount \ No newline at end of file