contracts/cultnet/cultnet-distributed-database.md

@@ -146,6 +146,9 @@ A malformed put or delete is `message_invalid`, not a per-operation code: the pe
   commit's changes as one commit, so the replica judges the result (a swap of two unique-index values) and not the state
   between records. A replica whose cache refuses a commit is faulted, never silent. `commitId` is opaque and valid only
   within the subscription. The C# subscription server and database server tag commits; other runtimes adopt next.
+- The C# writer sets every key of a `database_change_raw` message, nil where unset: a removed change carries `document`
+  nil, an added or updated one `recordKey` and `schemaId` nil. The schema accepts those nils; an added or updated change
+  still needs a document, and a removed one a `recordKey` and `schemaId`.
 - Client-owned input documents may be predicted locally only inside explicit
   `CultNetClientAuthorityScope` declarations.
 - Predicted input state is not authoritative shared state. The shard log still

contracts/cultnet/cultnet.database-change-raw.schema.json

@@ -10,17 +10,29 @@
     "messageId": { "type": "string", "minLength": 1 },
     "subscriptionId": { "type": "string", "minLength": 1 },
     "changeKind": { "enum": ["added", "updated", "removed"] },
-    "document": { "$ref": "./cultnet.raw-document-record.schema.json" },
-    "recordKey": { "type": "string", "minLength": 1 },
-    "schemaId": { "type": "string", "minLength": 1 },
+    "document": {
+      "oneOf": [{ "type": "null" }, { "$ref": "./cultnet.raw-document-record.schema.json" }]
+    },
+    "recordKey": { "type": ["string", "null"], "minLength": 1 },
+    "schemaId": { "type": ["string", "null"], "minLength": 1 },
     "commitId": { "type": ["string", "null"], "minLength": 1 },
     "commitCount": { "type": ["integer", "null"], "minimum": 2 }
   },
-  "allOf": [
+  "oneOf": [
     {
-      "if": { "properties": { "changeKind": { "const": "removed" } } },
-      "then": { "required": ["recordKey", "schemaId"] },
-      "else": { "required": ["document"] }
+      "properties": {
+        "changeKind": { "const": "removed" },
+        "recordKey": { "type": "string" },
+        "schemaId": { "type": "string" }
+      },
+      "required": ["recordKey", "schemaId"]
+    },
+    {
+      "properties": {
+        "changeKind": { "enum": ["added", "updated"] },
+        "document": { "$ref": "./cultnet.raw-document-record.schema.json" }
+      },
+      "required": ["document"]
     }
   ]
 }

packages/cultcache-ts/src/cult-cache.ts

@@ -1,4 +1,5 @@
 import {
+  declaresRole,
   decodeDocumentPayload,
   documentPayloadBytes,
   documentPayloadFromBytes,
@@ -240,6 +241,10 @@ export class CultCache {
         throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`);
       }
 
+      if (declaresRole(definition, indexName)) {
+        throw new Error(`Index "${indexName}" of ${definition.type} is declared as a role; a role is never a unique index.`);
+      }
+
       const registered = this.#requireDefinition(definition);
       const indexAccessors = new Map(registered.indexAccessors);
       indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor));

packages/cultcache-ts/src/document.ts

@@ -131,6 +131,12 @@ export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition
   return documentPayloadFromBytes(bytes);
 }
 
+// Whether the definition declares `name` as a role alias (either `roles` form).
+export function declaresRole(definition: AnyCultCacheDocumentDefinition, name: string): boolean {
+  const roles = definition.roles;
+  return Array.isArray(roles) ? roles.some((entry) => entry.name === name) : roles ? Object.hasOwn(roles, name) : false;
+}
+
 // The schema members of a document: the ones it declares, else those its codec derives, else none.
 export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinition): readonly CultCacheSchemaCatalogMember[] {
   if (definition.members || !definition.codec) {

packages/cultcache-ts/test/unique-index.test.ts

@@ -381,11 +381,21 @@ test("a refused load leaves the cache's prior state as it was, and a later good
   }
 });
 
-test("registerIndex over held duplicates installs nothing", async () => {
+test("registerIndex refuses a name the definition declares as a role", async () => {
   const document = item("item", "roles");
   const cache = memory(document);
   await cache.put(document, "a", { sku: "S", group: "g" });
   await cache.put(document, "b", { sku: "S", group: "h" });
+  await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: /declared as a role; a role is never a unique index/u });
+  assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined);
+  await cache.put(document, "c", { sku: "S", group: "i" });
+});
+
+test("registerIndex over held duplicates installs nothing", async () => {
+  const document = item("item", "none");
+  const cache = memory(document);
+  await cache.put(document, "a", { sku: "S", group: "g" });
+  await cache.put(document, "b", { sku: "S", group: "h" });
   await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: refusal("a", "b", "tests.item", "sku") });
   assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined, "no map was installed");
   await cache.put(document, "c", { sku: "S", group: "i" });
@@ -468,7 +478,7 @@ test("a load judges every unique alias: a store collision on the second alias re
 });
 
 test("registerIndex of a second unique alias judges it, naming the two smallest keys whatever order they were written in", async () => {
-  const document = pair({ indexes: { a: "a" }, roles: { b: "b" } });
+  const document = pair({ indexes: { a: "a" } });
   const cache = memory(document);
   await cache.put(document, "z", { a: "1", b: "s" });
   await cache.put(document, "m", { a: "2", b: "s" });

packages/cultnet-py/src/cultnet_py/schema_catalog.py

@@ -429,9 +429,9 @@ def wire_message_schema_properties(schema_version: str) -> dict[str, Any]:
             "messageId": common["messageId"],
             "subscriptionId": common["subscriptionId"],
             "changeKind": {"enum": ["added", "updated", "removed"]},
-            "document": {"$ref": "#/$defs/rawDocumentRecord"},
-            "schemaId": common["schemaId"],
-            "recordKey": common["recordKey"],
+            "document": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/rawDocumentRecord"}]},
+            "schemaId": {"type": ["string", "null"], "minLength": 1},
+            "recordKey": {"type": ["string", "null"], "minLength": 1},
             "commitId": {"type": ["string", "null"], "minLength": 1},
             "commitCount": {"type": ["integer", "null"], "minimum": 2},
         },

packages/cultnet-py/tests/test_cultnet.py

@@ -2315,9 +2315,10 @@ class CultNetTests(unittest.TestCase):
 
         change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"])
         self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"])
-        self.assertIn("document", change_schema["properties"])
-        self.assertIn("schemaId", change_schema["properties"])
-        self.assertIn("recordKey", change_schema["properties"])
+        # The C# writer sets every key, nil where unset (a removed change's document, an added change's recordKey and schemaId).
+        self.assertEqual(change_schema["properties"]["document"]["oneOf"][0], {"type": "null"})
+        self.assertEqual(change_schema["properties"]["schemaId"], {"type": ["string", "null"], "minLength": 1})
+        self.assertEqual(change_schema["properties"]["recordKey"], {"type": ["string", "null"], "minLength": 1})
         self.assertEqual(change_schema["properties"]["commitId"], {"type": ["string", "null"], "minLength": 1})
         self.assertEqual(change_schema["properties"]["commitCount"], {"type": ["integer", "null"], "minimum": 2})
 

packages/cultnet-rs/tests/database_change_commit_vector.rs

@@ -0,0 +1,43 @@
+//! `tests/vectors/database-change-commit/two-change-commit.msgpack` is written only by the C# reference: two
+//! `cultnet.database_change_raw.v0` messages of one commit, every key present, nil where unset. cultnet-rs has no typed
+//! `database_change_raw` message yet, so this walks the stream with the same `rmpv` values its wire decoder builds on and
+//! pins the keys and values a Rust adopter will read. A rename of `commitId` or `commitCount` in the C# writer fails here.
+
+use rmpv::Value;
+use std::io::Cursor;
+
+fn messages() -> Vec<Value> {
+    let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../tests/vectors/database-change-commit/two-change-commit.msgpack");
+    let bytes = std::fs::read(path).expect("the vector is committed");
+    let mut cursor = Cursor::new(bytes.as_slice());
+    let mut out = Vec::new();
+    while (cursor.position() as usize) < bytes.len() {
+        out.push(rmpv::decode::read_value(&mut cursor).expect("a msgpack value"));
+    }
+    out
+}
+
+fn field<'a>(message: &'a Value, key: &str) -> &'a Value {
+    message
+        .as_map()
+        .and_then(|entries| entries.iter().find(|(k, _)| k.as_str() == Some(key)))
+        .map(|(_, v)| v)
+        .unwrap_or_else(|| panic!("the C# writer wrote no key {key}"))
+}
+
+#[test]
+fn a_two_change_commit_written_by_csharp_carries_its_commit_id_count_and_changes() {
+    let messages = messages();
+    assert_eq!(messages.len(), 2);
+    for message in &messages {
+        assert_eq!(field(message, "schemaVersion").as_str(), Some("cultnet.database_change_raw.v0"));
+        assert_eq!(field(message, "commitId").as_str(), Some("9f1c0f0e4b7a4c1f9d2e"));
+        assert_eq!(field(message, "commitCount").as_u64(), Some(2));
+        assert_eq!(field(message, "changeKind").as_str(), Some("removed"));
+        assert_eq!(field(message, "subscriptionId").as_str(), Some("sub-1"));
+        assert_eq!(field(message, "schemaId").as_str(), Some("schema-note"));
+        assert!(field(message, "document").is_nil(), "a removed change's document is nil");
+    }
+    let keys: Vec<_> = messages.iter().map(|m| field(m, "recordKey").as_str().unwrap().to_string()).collect();
+    assert_eq!(keys, ["note:a", "note:b"]);
+}

packages/cultnet-ts/test/database-change-commit-vector.test.ts

@@ -0,0 +1,39 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import Ajv2020 from "ajv/dist/2020.js";
+import { decodeMulti } from "@msgpack/msgpack";
+
+import { cultNetSchemas, parseCultNetMessage } from "../src/index";
+
+// tests/vectors/database-change-commit/two-change-commit.msgpack is written only by the C# reference: two removed
+// database_change_raw messages of one commit, every key present, nil where unset (document included).
+const vector = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "database-change-commit", "two-change-commit.msgpack");
+const changes = () => [...decodeMulti(readFileSync(vector))] as Record<string, unknown>[];
+
+test("a two-change commit written by C# decodes with its commitId, commitCount and changes", () => {
+  const messages = changes().map((message) => parseCultNetMessage(structuredClone(message)));
+  assert.equal(messages.length, 2);
+  for (const message of messages) {
+    assert.equal(message.schemaVersion, "cultnet.database_change_raw.v0");
+    assert.equal((message as { commitId?: unknown }).commitId, "9f1c0f0e4b7a4c1f9d2e");
+    assert.equal((message as { commitCount?: unknown }).commitCount, 2);
+    assert.equal((message as { changeKind?: unknown }).changeKind, "removed");
+    assert.equal((message as { subscriptionId?: unknown }).subscriptionId, "sub-1");
+    assert.equal((message as { schemaId?: unknown }).schemaId, "schema-note");
+  }
+  assert.deepEqual(messages.map((message) => (message as { recordKey?: unknown }).recordKey), ["note:a", "note:b"]);
+});
+
+test("the database_change_raw schema accepts the C# vector's removed changes (document: nil) and still refuses an added change without a document", () => {
+  const ajv = new Ajv2020({ allErrors: true, strict: false });
+  ajv.addSchema(cultNetSchemas.rawDocumentRecordSchema);
+  const validate = ajv.compile(cultNetSchemas.databaseChangeRawSchema);
+  for (const change of changes()) {
+    assert.equal(change.document, null, "the C# writer wrote the key, nil");
+    assert.equal(validate(change), true, JSON.stringify(validate.errors));
+    assert.equal(validate({ ...change, changeKind: "added" }), false, "an added change needs a document");
+  }
+});

tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs

@@ -1,5 +1,7 @@
 #nullable enable
 using System;
+using System.Buffers;
+using System.Collections.Generic;
 using System.IO;
 using System.Text.Json;
 using System.Threading.Tasks;
@@ -246,6 +248,39 @@ namespace GameCult.Networking.Tests
             Assert.That(Convert.ToHexString(CultNetSchemaMessageSerialization.Serialize(message)), Is.EqualTo("85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D93E6D6573736167655F696E76616C69643A2052617720707574206D65737361676573206D75737420636F6E7461696E206120646F63756D656E74206D61702EAB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A8646F63756D656E74A576616C7565C0A461734F66C0A763757272656E74C0"));
         }
 
+        // The C# writer sets every key (nil where unset), so a removed change carries document: nil, and an added change
+        // carries recordKey/schemaId: nil. The schema describes those bytes; it still refuses an added change with no document.
+        private static List<JsonElement> CommitVectorChanges()
+        {
+            var path = Path.Combine(RepoRoot(), "tests", "vectors", "database-change-commit", "two-change-commit.msgpack");
+            var reader = new MessagePackReader(File.ReadAllBytes(path));
+            var changes = new List<JsonElement>();
+            while (!reader.End)
+                changes.Add(JsonDocument.Parse(MessagePackSerializer.ConvertToJson(reader.ReadRaw().ToArray())).RootElement);
+            return changes;
+        }
+
+        [Test]
+        public void DatabaseChangeRaw_TheCommitVectorsRemovedChangesWithANilDocument_MatchTheSchema()
+        {
+            var validator = new MiniJsonSchemaValidator(SchemaDir());
+            var changes = CommitVectorChanges();
+            Assert.That(changes, Has.Count.EqualTo(2));
+            foreach (var change in changes)
+            {
+                Assert.That(change.GetProperty("document").ValueKind, Is.EqualTo(JsonValueKind.Null), "the C# writer wrote the key, nil");
+                validator.AssertValid(change, "cultnet.database-change-raw.schema.json");
+            }
+        }
+
+        [Test]
+        public void DatabaseChangeRaw_AnAddedChangeWithANilDocument_IsRefused()
+        {
+            var validator = new MiniJsonSchemaValidator(SchemaDir());
+            var removed = CommitVectorChanges()[0].GetRawText().Replace("\"removed\"", "\"added\"");
+            Assert.Throws<InvalidOperationException>(() => validator.AssertValid(JsonDocument.Parse(removed).RootElement, "cultnet.database-change-raw.schema.json"));
+        }
+
         private static void AssertErrorMatchesSchema(CultNetErrorMessage message)
         {
             var validator = new MiniJsonSchemaValidator(SchemaDir());