packages/cultcache-py/src/cultcache_py/cache.py

@@ -234,6 +234,8 @@ class CultCache:
     @_mutating
     def register_index(self, document: DocumentDefinition[Any], index: str, extractor: str | Any) -> None:
         self._assert_registered(document)
+        if index in document.roles:
+            raise CultCacheError(f"Index '{index}' of {document.type} is declared as a role; a role is never a unique index.")
         index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()}
         index_extractors.setdefault(document.type, {})[index] = extractor
         self._install_lookups(self._state.name_extractors, index_extractors)
@@ -243,7 +245,11 @@ class CultCache:
     ) -> None:
         """Derives every lookup under the candidate extractors, then installs extractors and lookups
         together. An extractor that raises on a held value installs nothing."""
-        names, indexes, lookup_keys = self._derive_lookups(self._state.values, name_extractors, index_extractors)
+        names, indexes, lookup_keys = self._derive_lookups(
+            [(type, key, value) for type, values in self._state.values.items() for key, value in values.items()],
+            name_extractors,
+            index_extractors,
+        )
         (
             self._state.name_extractors,
             self._state.index_extractors,
@@ -311,6 +317,7 @@ class CultCache:
         structures, running every check and every user extractor; the cache's state is replaced
         only once that build finishes, so a refused load admits nothing."""
         loaded_values: dict[str, dict[str, Any]] = {}
+        read_order: dict[tuple[str, str], Any] = {}  # (type, key) -> value, in the order the stores yielded them
         loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {}
         loaded_legacy_global_keys: dict[str, str] = {}
         routes, generic = self._state.stores_by_type, self._state.generic_store
@@ -349,9 +356,10 @@ class CultCache:
                 envelope = self._stamped(document, envelope)
                 value = document.decode_payload(envelope.payload)
                 loaded_values.setdefault(envelope.type, {})[envelope.key] = value
+                read_order[(envelope.type, envelope.key)] = value
                 loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope
         loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups(
-            loaded_values, self._state.name_extractors, self._state.index_extractors
+            [(type, key, value) for (type, key), value in read_order.items()], self._state.name_extractors, self._state.index_extractors
         )
         # Plain attribute assignments: nothing fallible runs once the swap begins.
         (
@@ -615,24 +623,22 @@ class CultCache:
 
     def _derive_lookups(
         self,
-        values_by_type: dict[str, dict[str, Any]],
+        records: list[tuple[str, str, Any]],
         name_extractors: dict[str, str | Any],
         index_extractors: dict[str, dict[str, str | Any]],
     ) -> tuple[_Names, _Indexes, _LookupKeysByType]:
-        """Builds name and index lookups for the given values in fresh dicts, touching no cache state."""
+        """Builds name and index lookups for the given (type, key, value) records, judged in the order given, in fresh dicts, touching no cache state."""
         names: _Names = {}
         indexes: _Indexes = {}
         lookup_keys: _LookupKeysByType = {}
-        for type, values in values_by_type.items():
-            name_extractor = name_extractors.get(type)
-            type_index_extractors = index_extractors.get(type, {})
-            for key, value in values.items():
-                lookup_keys.setdefault(type, {})[key] = self._lookup_keys_for(value, name_extractor, type_index_extractors)
+        landing: list[tuple[str, str, _LookupKeys]] = []
+        for type, key, value in records:
+            keys = self._lookup_keys_for(value, name_extractors.get(type), index_extractors.get(type, {}))
+            lookup_keys.setdefault(type, {})[key] = keys
+            landing.append((type, key, keys))
         # Judged whole, over every type at once and on the file alone (never against what the cache held): a held value can
         # have been mutated in place since it was admitted, and another process may have moved a value between records.
-        self._refuse_index_sharing(
-            [(type, key, keys) for type, by_key in lookup_keys.items() for key, keys in by_key.items()], {}, ()
-        )
+        self._refuse_index_sharing(landing, {}, ())
         for type in lookup_keys:
             for key, keys in lookup_keys.get(type, {}).items():
                 if keys.name is not None:

packages/cultcache-py/tests/test_unique_index.py

@@ -544,6 +544,41 @@ class BatchAndOrderTests(unittest.TestCase):
             cache.pull_all_backing_stores()
         self.assertEqual(str(caught.exception), refusal("s:1", "s:2", "second", "alpha"))
 
+    def test_aliases_sort_by_utf16_code_unit_not_code_point(self) -> None:
+        astral, bmp = "k\U0001F600", "k\uff01"  # code points: bmp < astral; UTF-16 units: D83D < FF01
+        document = item_doc(indexes={bmp: "sku", astral: "serial"}, roles={})
+        cache = cache_of(document)
+        rows = [Item(sku="S", serial="T"), Item(sku="S", serial="T")]
+        with self.assertRaises(CultCacheError) as caught:
+            cache.put_envelopes(document, [envelope(document, f"item:{n}", row) for n, row in enumerate(rows)])
+        self.assertEqual(str(caught.exception), refusal("item:0", "item:1", "item", astral))
+
+    def test_a_load_judges_records_in_the_order_the_file_yields_them(self) -> None:
+        alpha, beta = item_doc("alpha", indexes={"k": "sku"}, roles={}), item_doc("beta", indexes={"k": "sku"}, roles={})
+        loose_alpha, loose_beta = item_doc("alpha", indexes={}, roles={}), item_doc("beta", indexes={}, roles={})
+        store = ListStore(
+            [
+                envelope(loose_alpha, "a:0", Item(sku="Q")),
+                envelope(loose_beta, "b:1", Item(sku="S")),
+                envelope(loose_beta, "b:2", Item(sku="S")),
+                envelope(loose_alpha, "a:1", Item(sku="P")),
+                envelope(loose_alpha, "a:2", Item(sku="P")),
+            ]
+        )
+        cache = cache_of(alpha, beta, store=store)
+        with self.assertRaises(CultCacheError) as caught:
+            cache.pull_all_backing_stores()
+        self.assertEqual(str(caught.exception), refusal("b:1", "b:2", "beta", "k"))
+
+    def test_register_index_refuses_an_alias_the_definition_declares_as_a_role(self) -> None:
+        document = item_doc()
+        cache = cache_of(document)
+        with self.assertRaises(CultCacheError):
+            cache.register_index(document, "group", "group")
+        cache.put_envelopes(
+            document, [envelope(document, "item:1", Item(group="guide")), envelope(document, "item:2", Item(group="guide"))]
+        )
+
 
 class LoadFreshnessTests(unittest.TestCase):
     def test_a_load_is_judged_on_the_file_alone_not_against_the_held_map(self) -> None:

packages/cultmesh-py/tests/test_unique_index_watch.py

@@ -36,6 +36,16 @@ class WatchByIndexTests(unittest.TestCase):
                     self.assertEqual(misheard, [])
                     self.assertIsNone(node.database.get_by_index(document, "price", str_text))
 
+    def test_a_watch_hears_a_record_leaving_a_value_by_the_canonical_text(self) -> None:
+        document = define_database_entry_type("mesh.leaving", [("price", 0)], indexes={"price": "price"})
+        node = CultMesh.create_node(runtime_id="mesh-watch-leaving")
+        node.database.register_document(document)
+        node.database.put(document, "item:1", {"price": 2.0})
+        heard: list[str] = []
+        node.database.watch_by_index(document, "price", "2", lambda change: heard.append(change.record_key))
+        node.database.put(document, "item:1", {"price": 3.0})
+        self.assertEqual(heard, ["item:1"])
+
     def test_a_blank_value_is_never_indexed_so_never_watched(self) -> None:
         document = define_database_entry_type("mesh.blank", [("sku", 0)], indexes={"sku": "sku"})
         node = CultMesh.create_node(runtime_id="mesh-watch-blank")

packages/cultnet-py/tests/test_cultnet.py

@@ -13,6 +13,7 @@ from datetime import UTC, datetime
 from pathlib import Path
 from typing import Callable
 from uuid import uuid4
+import msgpack
 from cultcache_py import CultCache, define_database_entry_type
 from cultnet_py.benchmark import run_benchmark
 from cultnet_py.compare_csharp import (
@@ -1600,25 +1601,13 @@ class CultNetTests(unittest.TestCase):
             with self.subTest(field=field, bad=bad), self.assertRaisesRegex(ValueError, field):
                 CultNetDatabaseChange.from_wire({**base, field: bad})
 
-    def test_a_two_change_commit_from_the_csharp_server_parses(self) -> None:
-        # CultNetDatabaseServer.DeliverCommit (CultNetDatabaseServer.cs:791-797) tags every change of a commit with its
-        # commitId and commitCount = the number of changes delivered, so a swap of two records arrives with commitCount 2.
-        # Shaped as the C# [MessagePackObject] writes it: every key present, nil where unset.
-        swap = [
-            {
-                "schemaVersion": "cultnet.database_change_raw.v0",
-                "messageId": f"change-{index}",
-                "subscriptionId": "sub-1",
-                "changeKind": "updated",
-                "document": None,
-                "recordKey": key,
-                "schemaId": "schema-note",
-                "commitId": "9f1c0f0e4b7a4c1f9d2e",
-                "commitCount": 2,
-            }
-            for index, key in enumerate(("note:a", "note:b"))
-        ]
-        changes = [CultNetDatabaseChange.from_wire(message) for message in swap]
+    def test_a_two_change_commit_written_by_csharp_parses(self) -> None:
+        # tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, written by the
+        # C# serializer (CultNetReplicaCommitTests), so a rename or retyping of commitId/commitCount fails here.
+        vector = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "database-change-commit" / "two-change-commit.msgpack"
+        messages = list(msgpack.Unpacker(io.BytesIO(vector.read_bytes()), raw=False))
+        self.assertEqual(len(messages), 2)
+        changes = [CultNetDatabaseChange.from_wire(message) for message in messages]
         self.assertEqual([change.commit_count for change in changes], [2, 2])
         self.assertEqual({change.commit_id for change in changes}, {"9f1c0f0e4b7a4c1f9d2e"})
         self.assertEqual([change.to_wire()["commitCount"] for change in changes], [2, 2])

tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs

@@ -225,6 +225,39 @@ namespace GameCult.Networking.Tests
             Assert.That(run.Faults, Has.Count.EqualTo(1));
         }
 
+        [Test]
+        public void ATwoChangeCommitIsWrittenAsTheVectorOtherRuntimesDecode()
+        {
+            // tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, as the C#
+            // serializer writes them, concatenated. Written with CULTCACHE_WRITE_VECTORS=1; every other run compares bytes.
+            var commit = Enumerable.Range(0, 2).Select(index => new CultNetDatabaseChangeRawMessage
+            {
+                MessageId = $"change-{index}",
+                SubscriptionId = "sub-1",
+                ChangeKind = "removed",
+                RecordKey = index == 0 ? "note:a" : "note:b",
+                SchemaId = "schema-note",
+                CommitId = "9f1c0f0e4b7a4c1f9d2e",
+                CommitCount = 2
+            });
+            var bytes = commit.SelectMany(message => CultNetSchemaMessageSerialization.Serialize(message)).ToArray();
+
+            var directory = TestContext.CurrentContext.TestDirectory;
+            while (directory != null && !System.IO.Directory.Exists(System.IO.Path.Combine(directory, "tests", "vectors")))
+                directory = System.IO.Path.GetDirectoryName(directory);
+            Assert.That(directory, Is.Not.Null, "tests/vectors is above the test directory");
+            var path = System.IO.Path.Combine(directory!, "tests", "vectors", "database-change-commit", "two-change-commit.msgpack");
+            if (Environment.GetEnvironmentVariable("CULTCACHE_WRITE_VECTORS") == "1")
+            {
+                System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!);
+                System.IO.File.WriteAllBytes(path, bytes);
+                return;
+            }
+
+            Assert.That(System.IO.File.Exists(path), Is.True, "the vector is committed");
+            Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(System.IO.File.ReadAllBytes(path))));
+        }
+
         [Test]
         public void AnOlderPeerReadsTheChangeAndIgnoresTheCommitFields()
         {

tests/vectors/database-change-commit/README.md

@@ -0,0 +1,10 @@
+# database-change-commit vectors
+
+Written only by the C# reference, `ATwoChangeCommitIsWrittenAsTheVectorOtherRuntimesDecode` in
+`tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs`, with `CULTCACHE_WRITE_VECTORS=1`. Every other run
+compares the bytes.
+
+- `two-change-commit.msgpack`: two `cultnet.database_change_raw.v0` messages (`removed`, `note:a` and `note:b`)
+  concatenated, each as the C# `[MessagePackObject]` writes it (every key present, nil where unset), both carrying
+  `commitId` `9f1c0f0e4b7a4c1f9d2e` and `commitCount` 2. Read it as a msgpack stream. Python walks it in
+  `packages/cultnet-py/tests/test_cultnet.py`.

tests/vectors/database-change-commit/two-change-commit.msgpack

@@ -0,0 +1 @@
+‰­schemaVersion¾cultnet.database_change_raw.v0©messageId¨change-0®subscriptionId¥sub-1ªchangeKind§removed¨documentÀ©recordKey¦note:a¨schemaId«schema-note¨commitId´9f1c0f0e4b7a4c1f9d2e«commitCount‰­schemaVersion¾cultnet.database_change_raw.v0©messageId¨change-1®subscriptionId¥sub-1ªchangeKind§removed¨documentÀ©recordKey¦note:b¨schemaId«schema-note¨commitId´9f1c0f0e4b7a4c1f9d2e«commitCount