From 621e05d552f0d98786695b4e68c107c420d3285e Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:41:18 +0000 Subject: [PATCH 001/117] cultmesh-ts: end a RUDP session after 8 consecutive unreadable frames or authority-refused deletes Heimdall's parity ruling. The refusal is still answered and the session kept below the cap; the frame that reaches CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES is answered too, then the session ends. The write-authority decision is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/CHANGELOG.md | 1 + packages/cultmesh-ts/src/index.ts | 38 ++++++++++-- .../test/remote-write-authority.test.ts | 58 +++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index 22d99904..d06e753b 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following ### Added +- `createRudpDocumentServer` ends a session after `CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES` (8) consecutive undecodable frames or consecutive deletes the write authority refuses; fewer are answered and the session is kept. Any decoded frame resets the undecodable run, any decoded non-refused-delete frame resets the delete run. The same cap holds in every runtime. - `CultMeshCatalogValue`, exported. - `documentById` and `tryDocumentById` on `CultMeshDocumentCatalog`, which address one document by its document id when a catalog holds several documents of one type. diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index e8d526cc..a548e49f 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4121,6 +4121,14 @@ export interface CultMeshRudpDocumentServerOptions extends CultMeshRudpSocketOpt /** A write authority that has not answered in this long has refused the write. */ export const CULTMESH_RUDP_DOCUMENT_SERVER_DEFAULT_DECIDE_TIMEOUT_MS = 5_000; +/** + * How many consecutive refused frames of one kind a `createRudpDocumentServer` session may send before it is + * ended: undecodable frames (reset by any frame that decodes) and deletes the write authority refuses (reset by any + * decoded frame that is not a refused delete). Fewer are answered with a typed refusal and the session is kept; the + * frame that reaches this number is answered too, then the session ends. The same number is the cap in every runtime. + */ +export const CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES = 8; + /** Replies larger than one fragment are split into fragments of at most this many bytes. */ export const CULTMESH_RUDP_DOCUMENT_SERVER_DEFAULT_MAX_FRAGMENT_BYTES = 1200; @@ -5537,6 +5545,9 @@ export class CultMesh { closed: boolean; /** A packet it owed could never be sent, so the session ended. */ unsendable: boolean; + /** Consecutive undecodable frames, and consecutive deletes the authority refused. */ + unreadableInARow: number; + refusedDeletesInARow: number; work: Promise; }; const sessions = new Map(); @@ -5591,6 +5602,8 @@ export class CultMesh { connectPayload, closed: false, unsendable: false, + unreadableInARow: 0, + refusedDeletesInARow: 0, session: new CultNetRudpSession({ connectionId, initialSequence: options.initialSequence, @@ -5666,11 +5679,16 @@ export class CultMesh { // an ungranted peer hears the same refusal for it as for any other write, not a parse error. const write = readableWrite(decoded); if (decoded !== undefined && !write) throw error; - if (await writeRefused(record, write?.operation ?? "put", write?.schemaId ?? "", write?.recordKey ?? "", null, null)) return; - reportError(error); - sendSchemaMessage(record, cultNetMessageInvalidError("message", "CultMesh RUDP document server could not read the write.")); + record.unreadableInARow += 1; + if (!await writeRefused(record, write?.operation ?? "put", write?.schemaId ?? "", write?.recordKey ?? "", null, null)) { + reportError(error); + sendSchemaMessage(record, cultNetMessageInvalidError("message", "CultMesh RUDP document server could not read the write.")); + } + if (record.unreadableInARow >= CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES) endCappedSession(record); return; } + record.unreadableInARow = 0; + if (message.schemaVersion !== "cultnet.document_delete.v0") record.refusedDeletesInARow = 0; switch (message.schemaVersion) { case "cultnet.snapshot_request.v0": { if (!options.getCache) { @@ -5744,7 +5762,12 @@ export class CultMesh { } return; case "cultnet.document_delete.v0": { - if (await writeRefused(record, "delete", message.schemaId, message.recordKey, null, null)) return; + if (await writeRefused(record, "delete", message.schemaId, message.recordKey, null, null)) { + record.refusedDeletesInARow += 1; + if (record.refusedDeletesInARow >= CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES) endCappedSession(record); + return; + } + record.refusedDeletesInARow = 0; if (!options.onDocumentDeleteRaw) { sendSchemaMessage(record, cultNetTextError("CultMesh RUDP document server has no delete handler.")); return; @@ -5895,6 +5918,13 @@ export class CultMesh { sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); } + /** A session that reached the cap of consecutive refused frames: its last refusal is sent, then it ends. */ + function endCappedSession(record: SessionRecord): void { + const key = `${record.remote.address}:${record.remote.port}`; + if (sessions.get(key) !== record) return; + endRefusedSession(key, record); + } + /** The session owes its peer a packet that can never be sent as built. */ function endUnsendableSession(record: SessionRecord, error: Error): void { if (record.unsendable) return; diff --git a/packages/cultmesh-ts/test/remote-write-authority.test.ts b/packages/cultmesh-ts/test/remote-write-authority.test.ts index 46fc5268..c6955d23 100644 --- a/packages/cultmesh-ts/test/remote-write-authority.test.ts +++ b/packages/cultmesh-ts/test/remote-write-authority.test.ts @@ -16,6 +16,7 @@ import { } from "@gamecult/cultnet-ts"; import { CultMesh, + CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES, CultMeshGrantAllWrites, type CultMeshRemoteWriteAuthority, type CultMeshRemoteWriteDecision, @@ -505,3 +506,60 @@ test("a granted peer's unreadable write reaches onError and gets a malformed ref } finally { r.close(); } } }); + +const CAP = 8; // the cap every runtime shares; the exported constant must stay this number + +test("the consecutive-refused-frames cap is the number every runtime shares", () => { + assert.equal(CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES, CAP); +}); + +test("unreadable frames are answered and the session kept until eight in a row; any decoded frame resets the run", async () => { + const r = await rawRig(new Recording(grant)); + try { + const batch = async (frames: number) => { + for (let i = 0; i < frames; i++) r.send(Uint8Array.from([0xc1])); + await sleep(400); + }; + await batch(CAP - 1); + assert.equal(r.replies.length, CAP - 1, "each is answered"); + r.send(encodeWireValue(put(KEY_A))); + await sleep(250); + assert.deepEqual(r.puts.map((p) => p.recordKey), [KEY_A], "the session was kept through seven"); + await batch(CAP - 1); + assert.equal(r.replies.length, 2 * (CAP - 1), "the valid frame reset the run"); + r.send(Uint8Array.from([0xc1])); + await sleep(300); + assert.equal(r.replies.length, 2 * (CAP - 1) + 1, "the eighth consecutive one is answered too"); + assert.ok(r.replies.every((reply) => (reply as { code?: string }).code === "message_invalid")); + // The goodbye reached the peer: its client no longer takes a send. + assert.throws(() => r.send(encodeWireValue(put(KEY_B))), /before the session is connected/, "then the session ended"); + assert.deepEqual(r.puts.map((p) => p.recordKey), [KEY_A]); + } finally { r.close(); } +}); + +test("deletes the authority refuses are answered and the session kept until eight in a row; any other decoded frame resets the run", async () => { + const authority = new Recording((request) => request.recordKey === KEY_B ? { granted: true } : { granted: false }); + await using(await rig(authority), async (r) => { + const refuse = async (deletes: number) => { + for (let i = 0; i < deletes; i++) r.peer.send(del(KEY_A)); + await sleep(400); + }; + await refuse(CAP - 1); + assert.equal(r.replies.length, CAP - 1); + r.peer.send(del(KEY_B)); // granted: resets the run + await sleep(250); + assert.deepEqual(r.deletes.map((d) => d.recordKey), [KEY_B]); + await refuse(CAP - 1); + assert.equal(r.replies.length, 2 * (CAP - 1)); + r.peer.send(put(KEY_B)); // a decoded non-delete resets too + await sleep(250); + await refuse(CAP - 1); + assert.equal(r.replies.length, 3 * (CAP - 1), "a put reset the run"); + r.peer.send(del(KEY_A)); + await sleep(300); + assert.equal(r.replies.length, 3 * (CAP - 1) + 1, "the eighth consecutive refused delete is answered too"); + for (const reply of r.replies) assertRefused(reply, KEY_A); + assert.throws(() => r.peer.send(del(KEY_B)), /before the session is connected/, "then the session ended"); + assert.equal(r.deletes.length, 1); + }); +}); From b4979cb65b9ece5b7d77db26536b15594378c5fd Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:43:41 +0000 Subject: [PATCH 002/117] cultmesh-py: answer an unreadable RUDP frame and end the session after 8 consecutive refusals Heimdall's parity ruling. A frame that is no msgpack map used to be dropped silently; it is now answered message_invalid (field "message") and the session kept. MAX_CONSECUTIVE_REFUSED_FRAMES (8) consecutive unreadable frames, or consecutive deletes the write authority refuses, end the session after the capped frame is answered. The authority's decision is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 45 ++++++++-- packages/cultmesh-py/tests/test_cultmesh.py | 89 +++++++++++++++++++ 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index 69583c1c..a871a84f 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -59,6 +59,13 @@ from cultnet_py.cultmesh_contracts import ( ) _LOGGER = logging.getLogger(__name__) + +# How many consecutive refused frames of one kind a RUDP session may send before it is ended: undecodable +# frames (reset by any frame that decodes) and deletes the write authority refuses (reset by any decoded +# frame that is not a refused delete). Fewer are answered with a typed refusal and the session is kept; the +# frame that reaches this number is answered too, then the session ends. The same number is the cap in every +# runtime. +MAX_CONSECUTIVE_REFUSED_FRAMES = 8 _EPHEMERAL_PAIR_ATTEMPTS = 64 @@ -90,6 +97,8 @@ class _DatabaseSubscription: class _RudpPeerConnection: session: CultNetRudpSession subscriptions: dict[str, _DatabaseSubscription] = field(default_factory=dict) + undecodable_frames_in_a_row: int = 0 + refused_deletes_in_a_row: int = 0 @dataclass @@ -390,6 +399,7 @@ class CultMeshLocalServer: if result.disconnected: peers.pop(remote_addr, None) continue + capped = False try: for delivered in result.delivered: if delivered.channel_id != "schema": @@ -397,15 +407,30 @@ class CultMeshLocalServer: try: message = msgpack.unpackb(delivered.payload, raw=False) except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): - continue - if not isinstance(message, dict): - continue - responses = self._handle_connection_message( - message, - peer.subscriptions, - over_rudp=True, - peer=CultMeshServerPeer(f"{remote_addr[0]}:{remote_addr[1]}", "rudp"), - ) + message = None + if isinstance(message, dict): + peer.undecodable_frames_in_a_row = 0 + responses = self._handle_connection_message( + message, + peer.subscriptions, + over_rudp=True, + peer=CultMeshServerPeer(f"{remote_addr[0]}:{remote_addr[1]}", "rudp"), + ) + refused_delete = message.get("schemaVersion") == "cultnet.document_delete.v0" and any( + response.get("code") == "write_not_authorized" for response in responses + ) + peer.refused_deletes_in_a_row = peer.refused_deletes_in_a_row + 1 if refused_delete else 0 + refused, run = refused_delete, peer.refused_deletes_in_a_row + else: + # A frame that is no CultNet message is answered, not dropped, and the session kept. + peer.undecodable_frames_in_a_row += 1 + refused, run = True, peer.undecodable_frames_in_a_row + responses = [self._message_invalid({}, "message", "CultMesh RUDP server could not read the message.")] + if refused and run >= MAX_CONSECUTIVE_REFUSED_FRAMES: + # The capped frame's refusal goes out with the goodbye; the session ends. + self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=responses[0]) + capped = True + break for response in responses: if unsendable is None: unsendable = self._send_rudp_schema_frame( @@ -434,6 +459,8 @@ class CultMeshLocalServer: ) self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=refusal) continue + if capped: + continue if unsendable is None and ( packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered ): diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index b49b6ed9..4844cd7c 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -24,7 +24,9 @@ from cultcache_py import ( define_document_type, ) from cultcache_py.interop import INTEROP_SCHEMA_VERSION, interop_note_document +from cultmesh_py.server import MAX_CONSECUTIVE_REFUSED_FRAMES from cultmesh_py.verify import verify +from cultmesh_py.write_authority import CultMeshRemoteWriteDecision from cultnet_py import ( compute_simulation_claim_hash, CultNetClientAuthorityScope, @@ -4889,6 +4891,93 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(after["documents"], []) self.assertTrue(alive) + def _rudp_open(self, server: Any) -> Any: + return create_rudp_schema_transport( + host="127.0.0.1", + port=server.port, + connection_id=server.rudp_connection_id, + timeout_seconds=4.0, + max_fragment_bytes=1200, + ) + + def _rudp_answer(self, transport: Any, payload: bytes) -> dict[str, Any]: + transport.send("schema", payload) + return msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + + def _assert_rudp_session_ended(self, transport: Any, server: Any) -> None: + transport.send("schema", msgpack.packb(snapshot_request(message_id="after-cap").to_wire(), use_bin_type=True)) + with self.assertRaises(Exception): + transport.receive(timeout_seconds=1.0) + self.assertTrue(server._rudp_thread.is_alive()) + + def test_cultmesh_local_server_answers_unreadable_frames_and_ends_the_session_at_eight_in_a_row(self) -> None: + # An undecodable frame (not a msgpack map) is answered message_invalid and the session kept; + # eight in a row end it (the eighth is answered too); any decoded frame resets the run. + self.assertEqual(MAX_CONSECUTIVE_REFUSED_FRAMES, 8) + node = CultMesh.create_node(runtime_id="mesh-rudp-unreadable") + server = CultMesh.serve_node(node, write_authority=CultMeshGrantAllWrites()) + garbage = b"\xc1" + snapshot = msgpack.packb(snapshot_request(message_id="ok").to_wire(), use_bin_type=True) + transport = self._rudp_open(server) + try: + for _ in range(2): + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + reply = self._rudp_answer(transport, garbage) + self.assertEqual(reply["code"], "message_invalid") + self.assertEqual(reply["details"]["field"], "message") + self.assertEqual(self._rudp_answer(transport, snapshot)["messageId"], "ok") + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + self._rudp_answer(transport, garbage) + transport.send("schema", garbage) + eighth = msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + self.assertEqual(eighth["code"], "message_invalid") + self._assert_rudp_session_ended(transport, server) + finally: + transport.close() + server.stop() + + def test_cultmesh_local_server_ends_the_session_at_eight_refused_deletes_in_a_row(self) -> None: + class GrantOnlyKeyB: + def decide(self, request: Any) -> Any: + return CultMeshRemoteWriteDecision.grant() if request.record_key == "note:b" else CultMeshRemoteWriteDecision.refuse("no") + + document = self._rudp_bound_note() + node = CultMesh.create_node(runtime_id="mesh-rudp-deletes") + node.database.register_document(document) + server = CultMesh.serve_node(node, write_authority=GrantOnlyKeyB()) + + def delete(key: str) -> bytes: + return msgpack.packb({ + "schemaVersion": "cultnet.document_delete.v0", + "messageId": f"del-{key}", + "schemaId": document.schema_id, + "recordKey": key, + }, use_bin_type=True) + + snapshot = msgpack.packb(snapshot_request(message_id="ok").to_wire(), use_bin_type=True) + transport = self._rudp_open(server) + try: + def refused_run(count: int) -> None: + for _ in range(count): + reply = self._rudp_answer(transport, delete("note:a")) + self.assertEqual(reply["code"], "write_not_authorized") + self.assertEqual(reply["details"]["value"], "note:a") + + refused_run(MAX_CONSECUTIVE_REFUSED_FRAMES - 1) + # A granted delete resets the run (nothing is stored under it, so the store answers that). + self.assertEqual(self._rudp_answer(transport, delete("note:b"))["code"], "unregistered_document_delete") + refused_run(MAX_CONSECUTIVE_REFUSED_FRAMES - 1) + # So does any decoded non-delete frame. + self.assertEqual(self._rudp_answer(transport, snapshot)["messageId"], "ok") + refused_run(MAX_CONSECUTIVE_REFUSED_FRAMES - 1) + transport.send("schema", delete("note:a")) + eighth = msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + self.assertEqual(eighth["code"], "write_not_authorized") + self._assert_rudp_session_ended(transport, server) + finally: + transport.close() + server.stop() + def test_cultmesh_local_server_refuses_a_put_its_store_cannot_take_and_serves_on(self) -> None: # Well-formed puts the store refuses: a blank key, a payload that is not MessagePack, a # payload its schema cannot decode, a global document under another key. Each is refused From 89bbb277bd13b947f36c7c84d995a825143e46c4 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:47:08 +0000 Subject: [PATCH 003/117] GameCult.Networking: answer an undecodable RUDP frame message_invalid and end the session at 8 in a row Heimdall's parity ruling. The schema server ended a peer's session on the first frame it could not decode. It now answers message_invalid (field "message") and keeps the session; RudpCultNetSchemaServer.MaxConsecutiveRefusedFrames (8) consecutive undecodable frames end it after the last is answered, and any decoded frame resets the run. Handler faults still end the session at once. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetRudpSchemaServer.cs | 22 +++++- .../CultNetRudpSchemaServerPeerInputTests.cs | 79 +++++++++++++++---- 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index de8d0154..444e97e9 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; @@ -113,9 +114,18 @@ namespace GameCult.Networking private readonly CultNetRudpSocketTransportServer _transport; private readonly ConcurrentDictionary _handlers = new(); private readonly ConcurrentDictionary _schemaPeerAdapters = new(); + private readonly ConditionalWeakTable> _undecodableFramesInARow = new(); private bool _disposed; private ILogger _logger = new NullLogger(); + /// + /// How many consecutive undecodable frames a peer's session may send before it is ended. Fewer are + /// answered with a typed message_invalid and the session is kept; the frame that reaches this + /// number is answered too, then the session ends. Any frame that decodes resets the run. The same + /// number is the cap in every runtime. + /// + public const int MaxConsecutiveRefusedFrames = 8; + /// /// Gets or sets the logger used by the server. R-AM: the dispatch backstop in DispatchAsync /// logs every untyped fault through this instead of swallowing it silently. @@ -317,11 +327,21 @@ namespace GameCult.Networking } catch (Exception ex) { - // The peer sent something no runtime can read: the refusal is the peer's. + // The peer sent something no runtime can read: the refusal is the peer's. It is answered + // message_invalid and the session kept, until MaxConsecutiveRefusedFrames in a row end it. + var run = _undecodableFramesInARow.GetOrCreateValue(delivered.Peer); + run.Value++; + _transport.SendSchemaMessage(delivered.Peer, CultNetErrorMessage.ForMessageInvalid("message", "CultNet RUDP schema server could not read the message.")); + if (run.Value < MaxConsecutiveRefusedFrames) + { + Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); it was refused and that peer's session kept."); + return false; + } var ended = _transport.EndPeerSession(delivered.Peer, CultNetRudpSession.RefusedPacketReason); Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); {SessionOutcome(ended)}"); return false; } + _undecodableFramesInARow.GetOrCreateValue(delivered.Peer).Value = 0; if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) { diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs index b8de674f..a07b1771 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -14,8 +14,9 @@ namespace GameCult.Networking.Tests { /// /// A schema server's poll serves every peer whatever one peer sends. A message that cannot be - /// decoded or handled is that peer's failure: its session ends, the drop is counted and logged, - /// and the drain goes on to the other peers. + /// decoded is answered message_invalid and its session kept until MaxConsecutiveRefusedFrames in a + /// row end it; one that cannot be handled ends that peer's session. The fault is logged, and the + /// drain goes on to the other peers. /// public class CultNetRudpSchemaServerPeerInputTests { @@ -161,30 +162,74 @@ namespace GameCult.Networking.Tests } [Test] - public async Task APeerWhoseMessageCannotBeDecodedHasItsSessionEnded() + public async Task APeerWhoseMessageCannotBeDecodedIsAnsweredAndKept() { using var scene = await BadThenGood(Malformed()); - var dropped = scene.Server.Stats.PacketsDropped; try { await scene.Server.PollAvailableAsync(16); } catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } - // The frame was already acknowledged, so the session cannot be kept: the peer is told, - // the listener forgets it, and the drop is counted. - Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint })); - Assert.That(scene.Server.Peers.Select(peer => peer.RemoteEndPoint), Is.EqualTo(new[] { scene.Good.EndPoint })); - Assert.That(scene.Server.Stats.PacketsDropped, Is.EqualTo(dropped + 1)); - var goodbye = Drain(scene.Bad.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); - Assert.That(scene.Bad.Session.Receive(goodbye, 1).Disconnected, Is.True); + Assert.That(scene.Ended, Is.Empty); + Assert.That(scene.Server.Peers, Has.Count.EqualTo(2)); + var refusal = ErrorsReceivedBy(scene.Bad).Single(); + Assert.That(refusal.Code, Is.EqualTo("message_invalid")); + Assert.That(refusal.Details!.Field, Is.EqualTo("message")); Assert.That(Drain(scene.Good.Socket).Select(packet => packet.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); } - // The peer's malformed message and a well-formed one after it arrive in one datagram's - // delivery: the second is held for order, and both are released together. The session ends - // on the first, so the second belongs to no session and is never handled. + private static List ErrorsReceivedBy(Client client) + { + var errors = new List(); + foreach (var packet in Drain(client.Socket)) + { + var received = client.Session.Receive(packet, 1); + foreach (var frame in received.Delivered) + if (CultNetSchemaMessageSerialization.Deserialize(frame.Payload) is CultNetErrorMessage error) + errors.Add(error); + } + return errors; + } + + // The numbers are the shared cross-runtime cap, spelled out so changing the constant fails here. [Test] - public async Task ARefusedPeersFramesStillQueuedAreNotHandled() + public async Task EightConsecutiveUndecodableFramesEndTheSessionAfterTheEighthIsAnswered() + { + Assert.That(RudpCultNetSchemaServer.MaxConsecutiveRefusedFrames, Is.EqualTo(8)); + using var scene = await BadThenGood(Malformed()); + await scene.Server.PollAvailableAsync(16); + ErrorsReceivedBy(scene.Bad); + async Task Refuse(int count) + { + for (var i = 0; i < count; i++) + { + SendSchema(scene.Server, scene.Bad, Malformed()); + await scene.Server.PollAvailableAsync(16); + } + } + + await Refuse(6); // with the first, seven in a row + Assert.That(ErrorsReceivedBy(scene.Bad), Has.Count.EqualTo(6)); + Assert.That(scene.Ended, Is.Empty); + SendSchema(scene.Server, scene.Bad, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "resets-the-run" })); + await scene.Server.PollAvailableAsync(16); + Assert.That(scene.Served.Select(s => s.MessageId), Does.Contain("resets-the-run")); + await Refuse(7); + Assert.That(ErrorsReceivedBy(scene.Bad), Has.Count.EqualTo(7)); + Assert.That(scene.Ended, Is.Empty, "a decoded frame reset the run"); + + await Refuse(1); + Assert.That(ErrorsReceivedBy(scene.Bad).Select(e => e.Code), Is.EqualTo(new[] { "message_invalid" }), "the eighth is answered too"); + Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint })); + Assert.That(scene.Server.Peers.Select(peer => peer.RemoteEndPoint), Is.EqualTo(new[] { scene.Good.EndPoint })); + } + + // The peer's malformed message and a well-formed one after it arrive in one datagram's + // delivery: the second is held for order, and both are released together. The session is + // kept through the first, so the second is handled. + [Test] + public async Task AFrameQueuedBehindAnUndecodableOneIsStillHandled() { var logger = new CapturingLogger(); using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions @@ -205,8 +250,8 @@ namespace GameCult.Networking.Tests await server.PollAvailableAsync(16); - Assert.That(served, Is.Empty); - Assert.That(server.Peers, Is.Empty); + Assert.That(served, Is.EqualTo(new[] { "after-the-refused-one" })); + Assert.That(server.Peers, Has.Count.EqualTo(1)); Assert.That(logger.Errors, Has.Count.EqualTo(1)); bad.Socket.Dispose(); } From 23c7ce37382c8cb784f11e274881177fa7a59cc5 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:52:22 +0000 Subject: [PATCH 004/117] cultmesh-ts: count and answer a frame that decodes but is no message the parser takes A map with an unknown schemaVersion was rethrown: unanswered, uncounted toward the session cap. It is now answered message_invalid, reported, and counted like any other unreadable frame, so the cap holds for it. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/src/index.ts | 8 ++++-- .../test/remote-write-authority.test.ts | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index a548e49f..5479f14b 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -5678,9 +5678,13 @@ export class CultMesh { // A frame the server cannot read is judged as the write it looks like (a put when it does not even decode), so // an ungranted peer hears the same refusal for it as for any other write, not a parse error. const write = readableWrite(decoded); - if (decoded !== undefined && !write) throw error; record.unreadableInARow += 1; - if (!await writeRefused(record, write?.operation ?? "put", write?.schemaId ?? "", write?.recordKey ?? "", null, null)) { + if (decoded !== undefined && !write) { + // It decodes but is no message the parser takes (an unknown schemaVersion, a non-map) and is no write the + // authority could judge: answered and counted like any other unreadable frame. + reportError(error); + sendSchemaMessage(record, cultNetMessageInvalidError("message", "CultMesh RUDP document server could not read the message.")); + } else if (!await writeRefused(record, write?.operation ?? "put", write?.schemaId ?? "", write?.recordKey ?? "", null, null)) { reportError(error); sendSchemaMessage(record, cultNetMessageInvalidError("message", "CultMesh RUDP document server could not read the write.")); } diff --git a/packages/cultmesh-ts/test/remote-write-authority.test.ts b/packages/cultmesh-ts/test/remote-write-authority.test.ts index c6955d23..0cebbb8c 100644 --- a/packages/cultmesh-ts/test/remote-write-authority.test.ts +++ b/packages/cultmesh-ts/test/remote-write-authority.test.ts @@ -563,3 +563,30 @@ test("deletes the authority refuses are answered and the session kept until eigh assert.equal(r.deletes.length, 1); }); }); + +test("a frame that decodes but is no message the parser takes is answered, counted, and ended at eight in a row like any unreadable frame", async () => { + const r = await rawRig(new Recording(grant)); + const unknown = () => encodeWireValue({ schemaVersion: "cultnet.not-a-message.v0", messageId: "x" }); + try { + const batch = async (frames: number) => { + for (let i = 0; i < frames; i++) r.send(unknown()); + await sleep(400); + }; + await batch(CAP - 1); + assert.equal(r.replies.length, CAP - 1, "each is answered"); + assert.equal(r.errors.length, CAP - 1, "and reported"); + r.send(encodeWireValue(put(KEY_A))); + await sleep(250); + assert.deepEqual(r.puts.map((p) => p.recordKey), [KEY_A], "the session was kept through seven"); + await batch(CAP - 1); + assert.equal(r.replies.length, 2 * (CAP - 1), "the decoded put reset the run"); + r.send(unknown()); + await sleep(300); + assert.equal(r.replies.length, 2 * (CAP - 1) + 1, "the eighth is answered too"); + for (const reply of r.replies) { + assert.equal((reply as { code?: string }).code, "message_invalid"); + assert.equal((reply as { details: { field: string } }).details.field, "message"); + } + assert.throws(() => r.send(unknown()), /before the session is connected/, "then the session ended"); + } finally { r.close(); } +}); From bf816955371cae48187f4f82faf8e1624f7d7768 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:54:37 +0000 Subject: [PATCH 005/117] cultmesh-py: answer the capped frame reliably and end the session once the answer is acknowledged The 8th refusal was sent unreliable and unordered with only the first response, then the goodbye at once, so a lost datagram left the peer without it. All answers now go the normal reliable path; the session ends when the peer has acknowledged them (a lost one is resent) or after a 2 s grace, and serves nothing after the capped frame meanwhile. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 37 +++++++++++---- packages/cultmesh-py/tests/test_cultmesh.py | 47 +++++++++++++++++++ 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index a871a84f..ab5267a7 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -66,6 +66,9 @@ _LOGGER = logging.getLogger(__name__) # frame that reaches this number is answered too, then the session ends. The same number is the cap in every # runtime. MAX_CONSECUTIVE_REFUSED_FRAMES = 8 + +# How long a session whose capped frame was answered waits for the peer to acknowledge the answer before it ends anyway. +_RUDP_CAPPED_ANSWER_GRACE_MS = 2000 _EPHEMERAL_PAIR_ATTEMPTS = 64 @@ -99,6 +102,8 @@ class _RudpPeerConnection: subscriptions: dict[str, _DatabaseSubscription] = field(default_factory=dict) undecodable_frames_in_a_row: int = 0 refused_deletes_in_a_row: int = 0 + # Set once the peer's capped frame has been answered: the session ends when the answers are acknowledged. + ending_deadline_ms: int | None = None @dataclass @@ -355,6 +360,7 @@ class CultMeshLocalServer: if packet.connection_id != self.rudp_connection_id: continue + self._end_answered_peers(rudp_socket, peers) now_ms = _now_ms() peer = peers.get(remote_addr) if packet.packet_type == CultNetRudpPacketType.CONNECT: @@ -399,11 +405,13 @@ class CultMeshLocalServer: if result.disconnected: peers.pop(remote_addr, None) continue - capped = False try: for delivered in result.delivered: if delivered.channel_id != "schema": continue + if peer.ending_deadline_ms is not None: + # The capped frame was answered; nothing after it is served while the answer is delivered. + break try: message = msgpack.unpackb(delivered.payload, raw=False) except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): @@ -426,11 +434,6 @@ class CultMeshLocalServer: peer.undecodable_frames_in_a_row += 1 refused, run = True, peer.undecodable_frames_in_a_row responses = [self._message_invalid({}, "message", "CultMesh RUDP server could not read the message.")] - if refused and run >= MAX_CONSECUTIVE_REFUSED_FRAMES: - # The capped frame's refusal goes out with the goodbye; the session ends. - self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=responses[0]) - capped = True - break for response in responses: if unsendable is None: unsendable = self._send_rudp_schema_frame( @@ -440,6 +443,11 @@ class CultMeshLocalServer: msgpack.packb(response, use_bin_type=True), message_id=str(response.get("messageId") or ""), ) + if refused and run >= MAX_CONSECUTIVE_REFUSED_FRAMES: + # The capped frame is answered like any other, reliably. The session ends once the peer + # has acknowledged every answer (or the grace runs out): see _end_answered_peers. + peer.ending_deadline_ms = now_ms + _RUDP_CAPPED_ANSWER_GRACE_MS + break except Exception as error: # noqa: BLE001 - one peer's message never ends the thread # Whatever a peer's message makes the handling raise is that peer's failure. # The session recorded the frame's sequence, so it cannot be kept: a retransmit @@ -459,8 +467,6 @@ class CultMeshLocalServer: ) self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=refusal) continue - if capped: - continue if unsendable is None and ( packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered ): @@ -472,11 +478,26 @@ class CultMeshLocalServer: if unsendable is not None: self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) + def _end_answered_peers( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + ) -> None: + """Ends each session whose capped frame was answered once the peer has acknowledged every answer (a lost + answer is resent first), or when the grace has run out.""" + now_ms = _now_ms() + for remote_addr, peer in list(peers.items()): + if peer.ending_deadline_ms is None: + continue + if peer.session.outstanding_reliable_packet_count == 0 or now_ms >= peer.ending_deadline_ms: + self._end_refused_peer(rudp_socket, peers, remote_addr) + def _poll_rudp_resends( self, rudp_socket: socket.socket, peers: dict[tuple[str, int], _RudpPeerConnection], ) -> None: + self._end_answered_peers(rudp_socket, peers) now_ms = _now_ms() for remote_addr, peer in list(peers.items()): for packet in peer.session.due_resends(now_ms): diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 4844cd7c..d96c4ab2 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4936,6 +4936,53 @@ class CultMeshTests(unittest.TestCase): transport.close() server.stop() + def test_cultmesh_local_server_resends_a_lost_capped_answer_before_it_ends_the_session(self) -> None: + # The eighth refusal is answered reliably: the datagram carrying it is lost on the wire, and the + # peer still gets the answer (by resend) before the goodbye. An unreliable final answer, or a + # goodbye sent at once, would leave the peer without it. + node = CultMesh.create_node(runtime_id="mesh-rudp-lost-answer") + server = CultMesh.serve_node(node, write_authority=CultMeshGrantAllWrites()) + send = server._send_rudp_packet + dropped: list[Any] = [] + armed = threading.Event() + + def lossy(rudp_socket: Any, remote_addr: Any, packet: Any) -> Any: + if armed.is_set() and not dropped and packet.packet_type == CultNetRudpPacketType.DATA and b"message_invalid" in (packet.payload or b""): + dropped.append(packet) + return None + return send(rudp_socket, remote_addr, packet) + + server._send_rudp_packet = lossy + transport = self._rudp_open(server) + try: + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + self._rudp_answer(transport, b"\xc1") + armed.set() + reply = self._rudp_answer(transport, b"\xc1") + self.assertEqual(len(dropped), 1, "fixture: the first transmission of the last answer was lost") + self.assertEqual(reply["code"], "message_invalid") + self._assert_rudp_session_ended(transport, server) + finally: + transport.close() + server.stop() + + def test_cultmesh_local_server_serves_nothing_after_the_capped_frame(self) -> None: + node = CultMesh.create_node(runtime_id="mesh-rudp-after-cap") + server = CultMesh.serve_node(node, write_authority=CultMeshGrantAllWrites()) + transport = self._rudp_open(server) + try: + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + self._rudp_answer(transport, b"\xc1") + transport.send("schema", b"\xc1") + transport.send("schema", msgpack.packb(snapshot_request(message_id="behind-the-cap").to_wire(), use_bin_type=True)) + first = msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + self.assertEqual(first["code"], "message_invalid") + with self.assertRaises(Exception): + transport.receive(timeout_seconds=1.0) + finally: + transport.close() + server.stop() + def test_cultmesh_local_server_ends_the_session_at_eight_refused_deletes_in_a_row(self) -> None: class GrantOnlyKeyB: def decide(self, request: Any) -> Any: From 9e5edeca52b985c76950ea99dc8a65679dbbcd43 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 10:59:23 +0000 Subject: [PATCH 006/117] cultmesh-ts: resend the capped frame's answer until acknowledged, then end the session endCappedSession deleted the session right after the answer was sent, so the resend timer never saw it and a lost datagram left the peer without the answer. The session now stays until the peer has acknowledged every reply (a lost one is resent) or a 2 s grace runs out, serves nothing after the capped frame meanwhile, and then sends the goodbye. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/src/index.ts | 20 +++- .../test/remote-write-authority.test.ts | 93 +++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 5479f14b..b77edd9f 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4128,6 +4128,8 @@ export const CULTMESH_RUDP_DOCUMENT_SERVER_DEFAULT_DECIDE_TIMEOUT_MS = 5_000; * frame that reaches this number is answered too, then the session ends. The same number is the cap in every runtime. */ export const CULTMESH_RUDP_DOCUMENT_SERVER_MAX_CONSECUTIVE_REFUSED_FRAMES = 8; +/** How long a session whose capped frame was answered waits for the peer to acknowledge the answers before it ends anyway. */ +const CULTMESH_RUDP_DOCUMENT_SERVER_CAPPED_ANSWER_GRACE_MS = 2000; /** Replies larger than one fragment are split into fragments of at most this many bytes. */ export const CULTMESH_RUDP_DOCUMENT_SERVER_DEFAULT_MAX_FRAGMENT_BYTES = 1200; @@ -5548,6 +5550,8 @@ export class CultMesh { /** Consecutive undecodable frames, and consecutive deletes the authority refused. */ unreadableInARow: number; refusedDeletesInARow: number; + /** Set once the peer's capped frame is answered: the session ends when the answers are acknowledged, or at this time. */ + endingDeadlineMs?: number; work: Promise; }; const sessions = new Map(); @@ -5668,7 +5672,8 @@ export class CultMesh { record: SessionRecord, payload: Uint8Array, ): Promise { - if (record.closed) return; + // Nothing after the capped frame is served while its answer is delivered. + if (record.closed || record.endingDeadlineMs !== undefined) return; let decoded: unknown; let message: CultNetMessage; try { @@ -5922,11 +5927,15 @@ export class CultMesh { sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); } - /** A session that reached the cap of consecutive refused frames: its last refusal is sent, then it ends. */ + /** + * A session that reached the cap of consecutive refused frames: its last refusal went out like any reply + * (reliable, so a lost datagram is resent), and the session ends once the peer has acknowledged every reply or + * the grace runs out. The resend timer ends it. + */ function endCappedSession(record: SessionRecord): void { const key = `${record.remote.address}:${record.remote.port}`; if (sessions.get(key) !== record) return; - endRefusedSession(key, record); + record.endingDeadlineMs ??= Date.now() + CULTMESH_RUDP_DOCUMENT_SERVER_CAPPED_ANSWER_GRACE_MS; } /** The session owes its peer a packet that can never be sent as built. */ @@ -6062,6 +6071,11 @@ export class CultMesh { closeSession(record); continue; } + if (record.endingDeadlineMs !== undefined + && (record.session.outstandingReliablePacketCount === 0 || nowMs >= record.endingDeadlineMs)) { + endRefusedSession(key, record); + continue; + } for (const packet of record.session.dueResends(nowMs)) { sendPacket(record, packet); } diff --git a/packages/cultmesh-ts/test/remote-write-authority.test.ts b/packages/cultmesh-ts/test/remote-write-authority.test.ts index 0cebbb8c..3c35171e 100644 --- a/packages/cultmesh-ts/test/remote-write-authority.test.ts +++ b/packages/cultmesh-ts/test/remote-write-authority.test.ts @@ -1,11 +1,13 @@ // A peer's put or delete is applied only when the server's write authority grants it. Ports the behaviours of // cultmesh-py test_remote_write_authority.py (no forwarding: this server does not forward non-primary writes). import assert from "node:assert/strict"; +import { createSocket, type Socket } from "node:dgram"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import test from "node:test"; import { z } from "zod"; import { documentPayloadFromBytes, defineDocumentType } from "@gamecult/cultcache-ts"; +import { decodeRudpPacket } from "@gamecult/cultnet-ts"; import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { CultNetDocumentRegistry, @@ -590,3 +592,94 @@ test("a frame that decodes but is no message the parser takes is answered, count assert.throws(() => r.send(unknown()), /before the session is connected/, "then the session ended"); } finally { r.close(); } }); + +/** A UDP relay between a client and a server that drops the first (or, with `dropEvery`, every) server datagram carrying `payloadContains` once armed. */ +async function lossyRelay(serverPort: number, payloadContains: string, dropEvery = false) { + const dropped: unknown[] = []; + let armed = false; + let clientAddress: { port: number; address: string } | undefined; + const toClient: Socket = createSocket("udp4"); + const toServer: Socket = createSocket("udp4"); + toServer.on("message", (wire) => { + if (!clientAddress) return; + if (armed && (dropEvery || dropped.length === 0)) { + const packet = decodeRudpPacket(wire); + if (packet.packetType === "data" && Buffer.from(packet.payload ?? []).includes(payloadContains)) { + dropped.push(packet); + return; + } + } + toClient.send(wire, clientAddress.port, clientAddress.address); + }); + toClient.on("message", (wire, remote) => { + clientAddress = remote; + toServer.send(wire, serverPort, "127.0.0.1"); + }); + await new Promise((resolve) => toClient.bind(0, "127.0.0.1", resolve)); + await new Promise((resolve) => toServer.bind(0, "127.0.0.1", resolve)); + return { + port: toClient.address().port, + dropped, + arm: () => { armed = true; }, + close: () => { toClient.close(); toServer.close(); }, + }; +} + +async function cappedOverRelay(dropEvery: boolean, body: (x: { relay: Awaited>; client: { send(channel: string, payload: Uint8Array): void }; replies: CultNetMessage[] }) => Promise) { + const connectionId = nextConnection++; + const server = CultMesh.createRudpDocumentServer("authority-lossy", connectionId, { + documents: new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: note })]), + writeAuthority: new Recording(grant), bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, + onDocumentPutRaw: () => {}, onError: () => {}, + }); + await server.start(); + const relay = await lossyRelay(server.bind.port, "message_invalid", dropEvery); + const client = await CultMesh.createRudpClient("lossy", connectionId, `rudp://127.0.0.1:${relay.port}`, { resendDelayMs: 25, resendPollMs: 5 }); + try { + client.connect(); + const started = Date.now(); + while (!client.connected) { + if (Date.now() - started > 2_000) throw new Error("not connected"); + await sleep(5); + } + const replies: CultNetMessage[] = []; + new CultNetPeer(client, { wireContract: "cultnet.schema.v0" }).on("message", (m) => replies.push(m)); + for (let i = 0; i < CAP - 1; i++) client.send("schema", Uint8Array.from([0xc1])); + await sleep(300); + assert.equal(replies.length, CAP - 1); + relay.arm(); + client.send("schema", Uint8Array.from([0xc1])); + await body({ relay, client, replies }); + } finally { client.close(); server.close(); relay.close(); } +} + +test("the capped frame's answer is resent when its datagram is lost, and the goodbye follows only after it arrived", async () => { + await cappedOverRelay(false, async ({ relay, client, replies }) => { + await sleep(600); + assert.equal(relay.dropped.length, 1, "fixture: the first transmission of the last answer was lost"); + assert.equal(replies.length, CAP, "the lost answer was resent and arrived"); + assert.equal((replies[CAP - 1] as { code?: string }).code, "message_invalid"); + assert.throws(() => client.send("schema", Uint8Array.from([0xc1])), /before the session is connected/, "then the session ended"); + }); +}); + +test("a capped session whose answer is never acknowledged still ends, after the grace", async () => { + await cappedOverRelay(true, async ({ client, replies }) => { + await sleep(1200); + assert.equal(replies.length, CAP - 1, "fixture: the last answer never arrives"); + client.send("schema", Uint8Array.from([0xc1])); // still connected: the server is waiting for the acknowledgement + await sleep(1500); + assert.throws(() => client.send("schema", Uint8Array.from([0xc1])), /before the session is connected/, "the grace ran out and the goodbye went out"); + }); +}); + +test("nothing behind the capped frame is served while its answer is delivered", async () => { + const r = await rawRig(new Recording(grant)); + try { + for (let i = 0; i < CAP; i++) r.send(Uint8Array.from([0xc1])); + r.send(encodeWireValue(put(KEY_A))); + await sleep(500); + assert.equal(r.replies.length, CAP); + assert.deepEqual(r.puts, [], "the put behind the cap was not served"); + } finally { r.close(); } +}); From 03141bc93bad3e2e1269edb8092224f383773169 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:04:22 +0000 Subject: [PATCH 007/117] cultmesh-rs: send the capped frame's answer reliably and end the session once it is acknowledged The capped (8th) refusal went out unreliable and unordered with the goodbye at once, so a lost datagram left the peer without it. It is now queued like any response and resent until the peer acknowledges it; the session ends then (or after a 2 s grace) and serves nothing after the capped frame meanwhile. The answer, its resends and the poll that carried the capped frame acknowledge nothing, as every refusal. A reliable queue that cannot take the answer ends the session at once with the unreliable refusal, as before. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 101 ++++++++++++++- .../cultmesh-rs/tests/rudp_document_server.rs | 120 +++++++++++++++++- 2 files changed, 210 insertions(+), 11 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 1553168b..30f76273 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -25,6 +25,10 @@ const SHORTEST_MESSAGE_ID: &str = "0"; /// is answered too, then the session ends. The same number is the cap in every runtime. pub const MAX_CONSECUTIVE_REFUSED_FRAMES: u32 = 8; +/// How long a session whose capped frame was answered waits for the peer to acknowledge the answer +/// before it ends anyway. +const CAPPED_ANSWER_GRACE_MILLIS: u64 = 2_000; + /// The transport identity of one remote CultNet RUDP session. /// /// Connection identifiers are scoped to a remote socket address. Two clients @@ -424,6 +428,9 @@ struct SessionEntry { admitted_payload_bytes: usize, undecodable_frames_in_a_row: u32, refused_deletes_in_a_row: u32, + /// Set once the peer's capped frame has been answered: the session ends when the answer is acknowledged, + /// or at this time. + ending_deadline_monotonic_millis: Option, } /// A synchronous, multi-session CultNet RUDP raw-document server. @@ -612,6 +619,14 @@ where if frame.channel_id != "schema" { continue; } + // The capped frame was answered: nothing after it is served while the answer is delivered. + if self + .sessions + .get(&key) + .is_some_and(|entry| entry.ending_deadline_monotonic_millis.is_some()) + { + break; + } let decoded = decode_cultnet_message_from_slice( &frame.payload, CultNetWireContract::CultNetSchemaV0, @@ -664,7 +679,7 @@ where let refusal = rejection.refusal_message(); let packets = match self.sessions.remove(&key) { Some(mut entry) => { - let packets = self.refusal_packets(&mut entry.session, &refusal); + let packets = self.refusal_packets(&mut entry.session, &refusal, false); self.sessions.insert(key, entry); packets } @@ -679,6 +694,45 @@ where kept_rejection = Some(rejection); continue; } + // The capped frame is answered reliably, so a lost datagram is resent; the session ends once + // the peer has acknowledged the answer or the grace runs out (`end_answered_sessions`). When + // the answer cannot be queued, the session ends at once with the refusal as before. + if (is_unreadable || is_delete) + && run.is_some() + && !matches!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseSendFailed(_) + ) + { + let refusal = rejection.refusal_message(); + let packets = match self.sessions.remove(&key) { + Some(mut entry) => { + let packets = self.refusal_packets(&mut entry.session, &refusal, true); + if !packets.is_empty() { + entry.ending_deadline_monotonic_millis = + Some(now_monotonic + CAPPED_ANSWER_GRACE_MILLIS); + } + self.sessions.insert(key, entry); + packets + } + None => Vec::new(), + }; + if packets.is_empty() { + self.end_session(key, SessionEnd::Rejected(refusal))?; + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } + for mut packet in packets { + // Acknowledges nothing, as every refusal: the refused frame was not admitted. + packet.ack = 0; + packet.ack_mask = 0; + if let Some(error) = self.send_packet(key.remote_addr, &packet)? { + unsendable = Some(error); + break; + } + } + kept_rejection = Some(rejection); + break; + } // The only reply `receive` returns is a Pong, which delivers no // frame, so no earlier send of this poll can precede a rejection. // A response that could not be sent ended its session where the @@ -693,7 +747,11 @@ where } } - if packet.reliable && unsendable.is_none() { + let answering_capped_frame = self + .sessions + .get(&key) + .is_some_and(|entry| entry.ending_deadline_monotonic_millis.is_some()); + if packet.reliable && unsendable.is_none() && !answering_capped_frame { let ack = self .sessions .get_mut(&key) @@ -705,6 +763,7 @@ where if let Some(error) = unsendable { self.end_session(key, SessionEnd::Unsendable(error))?; } + self.end_answered_sessions()?; Ok(match kept_rejection { Some(rejection) => CultMeshRudpPollOutcome::ApplicationRejected(rejection), None => CultMeshRudpPollOutcome::Handled, @@ -722,10 +781,16 @@ where && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms }); let expired = before - self.sessions.len(); + self.end_answered_sessions()?; let mut resends = Vec::new(); for (key, entry) in &mut self.sessions { - for packet in entry.session.due_resends(now) { + for mut packet in entry.session.due_resends(now) { + if entry.ending_deadline_monotonic_millis.is_some() { + // The resent answer to a capped frame acknowledges nothing either. + packet.ack = 0; + packet.ack_mask = 0; + } resends.push((*key, packet)); } } @@ -744,6 +809,26 @@ where }) } + /// Ends each session whose capped frame was answered once the peer has acknowledged every answer (a lost + /// answer is resent first), or when the grace has run out. + fn end_answered_sessions(&mut self) -> Result<()> { + let now = self.clock.now_monotonic_millis(); + let due: Vec = self + .sessions + .iter() + .filter(|(_, entry)| { + entry.ending_deadline_monotonic_millis.is_some_and(|deadline| { + entry.session.outstanding_reliable_packet_count() == 0 || now >= deadline + }) + }) + .map(|(key, _)| *key) + .collect(); + for key in due { + self.end_session(key, SessionEnd::Refused)?; + } + Ok(()) + } + /// `Ok(false)` when the Connect was refused because the session cap is full; /// only a socket failure is an `Err`. fn accept_connection( @@ -795,6 +880,7 @@ where admitted_payload_bytes: 0, undecodable_frames_in_a_row: 0, refused_deletes_in_a_row: 0, + ending_deadline_monotonic_millis: None, }, ); accept @@ -1034,7 +1120,7 @@ where }; let (refusal, mut unsendable) = match end { SessionEnd::Refused => (Vec::new(), None), - SessionEnd::Rejected(refusal) => (self.refusal_packets(&mut entry.session, &refusal), None), + SessionEnd::Rejected(refusal) => (self.refusal_packets(&mut entry.session, &refusal, false), None), SessionEnd::Unsendable(error) => (Vec::new(), Some(error)), }; entry.session.reset_peer_state(); @@ -1057,11 +1143,12 @@ where Ok(()) } - /// The refusal a rejected peer is sent, as unreliable, unordered packets. + /// The refusal a rejected peer is sent: unreliable and unordered, or (`reliable`) queued like any response. fn refusal_packets( &self, session: &mut CultNetRudpSession, refusal: &CultNetMessage, + reliable: bool, ) -> Vec { encode_cultnet_message_to_vec(refusal, CultNetWireContract::CultNetSchemaV0) .and_then(|payload| { @@ -1069,8 +1156,8 @@ where "schema", payload, CultNetRudpSendOptions { - reliable: false, - ordered: false, + reliable, + ordered: reliable, sequenced: false, now_ms: self.clock.now_monotonic_millis(), reliable_expire_after_ms: None, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 466037c7..a1954955 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -343,14 +343,17 @@ fn refusal_seen_by( client: &mut CultNetRudpSocketTransportConnection, receipt: &CultNetRudpReliableSendReceipt, ) -> Result { - let (error, code, details) = typed_refusal_seen_by(client, receipt)?; + let (error, code, details) = typed_refusal_seen_by(client, receipt, None)?; assert_eq!((code, details), (None, None)); Ok(error) } +/// `server` is polled while the goodbye is awaited when the refusal ends its session only once the peer has +/// acknowledged it (the capped frame's answer). fn typed_refusal_seen_by( client: &mut CultNetRudpSocketTransportConnection, receipt: &CultNetRudpReliableSendReceipt, + mut server: Option<&mut Server>, ) -> Result<(String, Option, Option)> { let mut refusal = None; for _ in 0..2_000 { @@ -381,6 +384,9 @@ fn typed_refusal_seen_by( "a refusal must not acknowledge the refused message" ); for _ in 0..2_000 { + if let Some(server) = server.as_deref_mut() { + server.poll_once()?; + } client.receive_once()?; if !client.connected() { break; @@ -681,7 +687,7 @@ fn application_rejection_is_nonfatal_peer_scoped_refused_to_the_peer_and_unackno // The caller keeps the sink's text; the peer is sent fixed text, never it. // A refused put is typed write_not_authorized and names the record; nothing of it reached the sink. assert_eq!( - typed_refusal_seen_by(&mut publisher, &publish_receipt)?, + typed_refusal_seen_by(&mut publisher, &publish_receipt, None)?, ( "write_not_authorized: the catalog refused the document".to_string(), Some(CultNetErrorCode::WriteNotAuthorized), @@ -863,7 +869,7 @@ fn an_unreadable_frame_is_refused_typed_and_the_session_kept_until_eight_in_a_ro panic!("the eighth consecutive unreadable frame must be refused"); }; assert_eq!(rejection.operation, CultMeshRudpApplicationOperation::Unreadable); - let (error, code, _) = typed_refusal_seen_by(&mut garbler, &receipt)?; + let (error, code, _) = typed_refusal_seen_by(&mut garbler, &receipt, Some(&mut server))?; assert_eq!((error.as_str(), code), (message_invalid.2.as_str(), message_invalid.0)); drain_until_idle(&mut server)?; assert_eq!(server.session_count(), 1, "only the capped peer's session ended"); @@ -905,7 +911,7 @@ fn a_refused_delete_keeps_the_session_until_eight_in_a_row() -> Result<()> { }; assert_eq!(rejection.operation, CultMeshRudpApplicationOperation::DocumentDelete); assert_eq!(rejection.reason, CultMeshRudpRejectionReason::DeleteNotSupported); - let (error, code, details) = typed_refusal_seen_by(&mut deleter, &receipt)?; + let (error, code, details) = typed_refusal_seen_by(&mut deleter, &receipt, Some(&mut server))?; assert_eq!( (error, code, details), { @@ -1623,6 +1629,11 @@ impl RawPeer { /// Sends `message` reliably and in order; returns its sequences. fn send(&mut self, message: &CultNetMessage) -> Result> { let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; + self.send_payload(payload) + } + + /// Sends `payload` as one reliable, ordered schema frame; returns its sequences. + fn send_payload(&mut self, payload: Vec) -> Result> { let options = cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: true, @@ -1910,3 +1921,104 @@ fn a_response_that_is_too_large_to_send_is_named_emsgsize() -> Result<()> { ); Ok(()) } + +/// A peer on `server` that has sent seven unreadable frames, each answered and processed, and then the +/// eighth, whose answer is waiting at its socket. Returns the peer and the eighth frame's sequence. +fn peer_at_the_cap(server: &mut Server, connection_id: u32) -> Result<(RawPeer, u32)> { + let mut peer = RawPeer::connect(server, connection_id)?; + for _ in 0..7 { + peer.send_payload(garbage())?; + assert!(matches!( + server.poll_once()?, + CultMeshRudpPollOutcome::ApplicationRejected(_) + )); + for packet in peer.drain() { + peer.session.receive(&packet, 3)?; + } + } + let capped = peer.send_payload(garbage())?[0]; + assert!(matches!( + server.poll_once()?, + CultMeshRudpPollOutcome::ApplicationRejected(_) + )); + Ok((peer, capped)) +} + +fn is_goodbye(packet: &cultnet_rs::CultNetRudpPacket) -> bool { + packet.packet_type == cultnet_rs::CultNetRudpPacketType::Disconnect +} + +fn is_answer(packet: &cultnet_rs::CultNetRudpPacket) -> bool { + packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data && packet.reliable +} + +/// The capped frame's answer is reliable: its datagram is lost, the server resends it, and only once the +/// peer has acknowledged it does the goodbye follow. Neither transmission acknowledges the capped frame. +#[test] +fn a_lost_capped_answer_is_resent_and_the_goodbye_waits_for_its_acknowledgement() -> Result<()> { + let clock = Clock::new(81_000); + let mut server = server(Default::default(), clock.clone(), Sink::default(), Source::default())?; + let (mut peer, capped) = peer_at_the_cap(&mut server, 81)?; + + let first = peer.drain(); + assert!(first.iter().any(is_answer), "fixture: the answer was sent, and is lost here"); + assert!(!first.iter().any(is_goodbye), "no goodbye before the answer is acknowledged"); + assert!( + !first.iter().any(|packet| acknowledges(packet, capped)), + "nothing sent for the capped frame acknowledges it" + ); + assert_eq!(server.session_count(), 1); + + clock.set_monotonic(81_000 + 60); + server.maintain()?; + let resent = peer.drain(); + let answer = resent.iter().find(|packet| is_answer(packet)).expect("the lost answer is resent"); + assert!(!acknowledges(answer, capped), "the resent answer acknowledges nothing the peer sent"); + assert!(!resent.iter().any(is_goodbye), "still no goodbye"); + let delivered = peer.session.receive(answer, 4)?.delivered; + let Ok(CultNetMessage::Error { code, .. }) = + decode_cultnet_message_from_slice(&delivered[0].payload, CultNetWireContract::CultNetSchemaV0) + else { + panic!("the resent answer is a typed refusal"); + }; + assert_eq!(code, Some(CultNetErrorCode::MessageInvalid)); + + peer.send_packet(&peer.session.create_ack_for(answer.sequence))?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + assert!(peer.drain().iter().any(is_goodbye), "acknowledged: the goodbye follows"); + assert_eq!(server.session_count(), 0); + Ok(()) +} + +/// A peer that never acknowledges the answer is not kept for ever: the session ends after the grace. +#[test] +fn a_capped_session_whose_answer_is_never_acknowledged_ends_after_the_grace() -> Result<()> { + let clock = Clock::new(82_000); + let mut server = server(Default::default(), clock.clone(), Sink::default(), Source::default())?; + let (peer, _) = peer_at_the_cap(&mut server, 82)?; + peer.drain(); + + clock.set_monotonic(82_000 + 1_999); + server.maintain()?; + assert!(!peer.drain().iter().any(is_goodbye), "inside the grace"); + assert_eq!(server.session_count(), 1); + clock.set_monotonic(82_000 + 2_000); + server.maintain()?; + assert!(peer.drain().iter().any(is_goodbye), "the grace ran out"); + assert_eq!(server.session_count(), 0); + Ok(()) +} + +/// Nothing behind the capped frame is served while its answer is delivered. +#[test] +fn nothing_behind_the_capped_frame_is_served() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(83_000), sink.clone(), Source::default())?; + let (mut peer, _) = peer_at_the_cap(&mut server, 83)?; + peer.send_payload(valid_put("behind-the-cap"))?; + for _ in 0..3 { + server.poll_once()?; + } + assert!(sink.0.lock().unwrap().receipts.is_empty(), "the put behind the cap was not stored"); + Ok(()) +} From 0b1a468ddd54ce0154b44daa97f94697b1216c35 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:10:48 +0000 Subject: [PATCH 008/117] GameCult.Networking: resend the capped frame's answer until acknowledged, then end the session The schema server answered the 8th undecodable frame and ended the peer's session in the same call; ending clears the session's pending reliable packets, so a lost datagram was never resent. The session now stays until the peer has acknowledged every answer (a lost one is resent by PollResends) or a 2 s grace runs out, serves nothing after the capped frame meanwhile, and then sends the goodbye. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetRudpSchemaServer.cs | 15 ++- src/GameCult.Networking/CultNetTransport.cs | 45 +++++++++ .../CultNetRudpSchemaServerPeerInputTests.cs | 91 +++++++++++++++++++ 3 files changed, 149 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index 444e97e9..3378036a 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -126,6 +126,12 @@ namespace GameCult.Networking /// public const int MaxConsecutiveRefusedFrames = 8; + /// + /// How long a session whose capped frame was answered waits for the peer to acknowledge the answer + /// before it ends anyway. + /// + internal const long CappedAnswerGraceMs = 2000; + /// /// Gets or sets the logger used by the server. R-AM: the dispatch backstop in DispatchAsync /// logs every untyped fault through this instead of swallowing it silently. @@ -310,6 +316,9 @@ namespace GameCult.Networking { if (!string.Equals(delivered.Frame.ChannelId, "schema", StringComparison.Ordinal)) return false; + // The capped frame was answered: nothing after it is served while the answer is delivered. + if (delivered.Peer.EndingReason != null) + return false; // A peer's message that cannot be decoded (malformed bytes, an unknown schemaVersion) or // handled ends that peer's session, never the poll: the drain goes on serving every other @@ -337,8 +346,10 @@ namespace GameCult.Networking Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); it was refused and that peer's session kept."); return false; } - var ended = _transport.EndPeerSession(delivered.Peer, CultNetRudpSession.RefusedPacketReason); - Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); {SessionOutcome(ended)}"); + // The answer was sent reliably: the session ends once the peer has acknowledged it (a lost + // datagram is resent first), or after the grace. + _transport.EndPeerSessionWhenAnswered(delivered.Peer, CultNetRudpSession.RefusedPacketReason, CappedAnswerGraceMs); + Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); that peer's session ends once its answer is acknowledged."); return false; } _undecodableFramesInARow.GetOrCreateValue(delivered.Peer).Value = 0; diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index ac2c2952..da465326 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -2759,6 +2759,11 @@ namespace GameCult.Networking // under SessionGate by whichever thread's send found the failure; the polling thread reads // it when it retires the peer. internal byte[]? UnsendableReason { get; set; } + + // Set once the peer's capped frame has been answered: the session ends with this reason when the + // peer has acknowledged every answer, or at the deadline. Under SessionGate. + internal byte[]? EndingReason { get; set; } + internal long EndingDeadlineMs { get; set; } } /// @@ -2928,6 +2933,7 @@ namespace GameCult.Networking public bool TryReceiveOnce(out CultNetRudpSocketServerFrame? delivered) { RetireUnsendablePeers(); + EndAnsweredPeers(); if (_deliveredFrames.Count > 0) { delivered = _deliveredFrames.Dequeue(); @@ -3146,6 +3152,7 @@ namespace GameCult.Networking public void PollResends() { RetireUnsendablePeers(); + EndAnsweredPeers(); if (_socket.Poll(0, SelectMode.SelectRead)) return; foreach (var peer in _peers.Values.ToArray()) @@ -3244,6 +3251,44 @@ namespace GameCult.Networking return true; } + // Polling thread only. The application answered a peer's capped frame: the answer is reliable, so a + // lost datagram is resent, and the session ends with `reason` once the peer has acknowledged every + // answer or `graceMs` have passed (EndAnsweredPeers). Frames of the peer still queued are not delivered. + internal void EndPeerSessionWhenAnswered(CultNetRudpSocketServerPeer peer, byte[] reason, long graceMs) + { + if (peer == null) throw new ArgumentNullException(nameof(peer)); + if (reason == null) throw new ArgumentNullException(nameof(reason)); + lock (peer.SessionGate) + { + if (peer.Session.Ended || peer.EndingReason != null) + return; + peer.EndingReason = reason; + peer.EndingDeadlineMs = NowMs() + graceMs; + } + var kept = _deliveredFrames.Where(frame => !ReferenceEquals(frame.Peer, peer)).ToArray(); + _deliveredFrames.Clear(); + foreach (var frame in kept) + _deliveredFrames.Enqueue(frame); + } + + // Polling thread only. + private void EndAnsweredPeers() + { + foreach (var peer in _peers.Values.ToArray()) + { + byte[]? reason; + lock (peer.SessionGate) + { + reason = peer.EndingReason != null + && (peer.Session.OutstandingReliablePacketCount == 0 || NowMs() >= peer.EndingDeadlineMs) + ? peer.EndingReason + : null; + } + if (reason != null) + EndPeerSession(peer, reason); + } + } + // Polling thread only. A peer a new Connect already replaced was reported then, and its key now // names the peer that replaced it. private void RetireUnsendablePeers() diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs index a07b1771..e7900e7c 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -191,6 +191,95 @@ namespace GameCult.Networking.Tests return errors; } + // The bad peer acknowledges what it received and the server polls on. + private static async Task Acknowledge(Scene scene) + { + scene.Bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(scene.Bad.Session.CreateAck()), scene.Server.LocalEndPoint); + await scene.Server.PollAvailableAsync(16); + } + + // The bad peer has sent seven undecodable frames (BadThenGood sent the first), each answered. + private static async Task AtTheCap() + { + var scene = await BadThenGood(Malformed()); + await scene.Server.PollAvailableAsync(16); + ErrorsReceivedBy(scene.Bad); + for (var i = 0; i < 6; i++) + { + SendSchema(scene.Server, scene.Bad, Malformed()); + await scene.Server.PollAvailableAsync(16); + ErrorsReceivedBy(scene.Bad); + } + Assert.That(scene.Ended, Is.Empty); + return scene; + } + + // The capped frame's answer is lost on the wire: the server resends it, and the goodbye follows only + // once the peer has acknowledged it. + [Test] + public async Task ALostCappedAnswerIsResentAndTheGoodbyeWaitsForItsAcknowledgement() + { + using var scene = await AtTheCap(); + SendSchema(scene.Server, scene.Bad, Malformed()); + await scene.Server.PollAvailableAsync(16); + + var lost = Drain(scene.Bad.Socket); + Assert.That(lost.Where(p => p.PacketType == CultNetRudpPacketType.Data), Is.Not.Empty, "fixture: the answer was sent and is lost here"); + Assert.That(lost.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect), "no goodbye before the answer is acknowledged"); + Assert.That(scene.Ended, Is.Empty); + + var resent = new List(); + for (var attempt = 0; attempt < 40 && resent.Count == 0; attempt++) + { + await Task.Delay(50); + scene.Server.PollResends(); + resent = Drain(scene.Bad.Socket); + } + Assert.That(resent.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect), "still no goodbye"); + var answers = new List(); + foreach (var packet in resent) + foreach (var frame in scene.Bad.Session.Receive(packet, 1).Delivered) + if (CultNetSchemaMessageSerialization.Deserialize(frame.Payload) is CultNetErrorMessage error) + answers.Add(error); + Assert.That(answers.Select(e => e.Code), Is.EqualTo(new[] { "message_invalid" }), "the lost answer arrived by resend"); + Assert.That(scene.Ended, Is.Empty); + + await Acknowledge(scene); + Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint }), "acknowledged: the session ends"); + Assert.That(Drain(scene.Bad.Socket).Select(p => p.PacketType), Has.Some.EqualTo(CultNetRudpPacketType.Disconnect)); + } + + // A peer that never acknowledges the answer is not kept for ever: the session ends after the grace. + [Test] + public async Task ACappedSessionWhoseAnswerIsNeverAcknowledgedEndsAfterTheGrace() + { + using var scene = await AtTheCap(); + SendSchema(scene.Server, scene.Bad, Malformed()); + await scene.Server.PollAvailableAsync(16); + Drain(scene.Bad.Socket); + + await Task.Delay(1500); // the grace is two seconds + scene.Server.PollResends(); + Assert.That(scene.Ended, Is.Empty, "inside the grace"); + await Task.Delay(700); + scene.Server.PollResends(); + Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint }), "the grace ran out"); + } + + // Nothing behind the capped frame is served while its answer is delivered. + [Test] + public async Task NothingBehindTheCappedFrameIsServed() + { + using var scene = await AtTheCap(); + SendSchema(scene.Server, scene.Bad, Malformed()); + SendSchema(scene.Server, scene.Bad, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "behind-the-cap" })); + await scene.Server.PollAvailableAsync(16); + await scene.Server.PollAvailableAsync(16); + + Assert.That(scene.Served.Select(s => s.MessageId), Does.Not.Contain("behind-the-cap")); + } + // The numbers are the shared cross-runtime cap, spelled out so changing the constant fails here. [Test] public async Task EightConsecutiveUndecodableFramesEndTheSessionAfterTheEighthIsAnswered() @@ -221,6 +310,8 @@ namespace GameCult.Networking.Tests await Refuse(1); Assert.That(ErrorsReceivedBy(scene.Bad).Select(e => e.Code), Is.EqualTo(new[] { "message_invalid" }), "the eighth is answered too"); + Assert.That(scene.Ended, Is.Empty, "the session ends once the answer is acknowledged"); + await Acknowledge(scene); Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint })); Assert.That(scene.Server.Peers.Select(peer => peer.RemoteEndPoint), Is.EqualTo(new[] { scene.Good.EndPoint })); } From b2077fa2bfa4734f3151eeafede5f7a9c2c27848 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:21:08 +0000 Subject: [PATCH 009/117] cultmesh-rs: a Pong in the capped grace acknowledges nothing; the grace deadline saturates A Ping in the 2 s grace after the capped frame was answered with a Pong built from the session's ack state, which includes the capped frame. Every reply while the session is ending now zeroes its ack fields. The deadline no longer wraps at the end of the clock. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 15 ++++++-- .../cultmesh-rs/tests/rudp_document_server.rs | 34 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 30f76273..647589e0 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -605,7 +605,18 @@ where }; let mut unsendable = match result.reply { - Some(reply) => self.send_packet(key.remote_addr, &reply)?, + Some(mut reply) => { + // A reply (a Pong) in the grace after the capped frame acknowledges nothing either. + if self + .sessions + .get(&key) + .is_some_and(|entry| entry.ending_deadline_monotonic_millis.is_some()) + { + reply.ack = 0; + reply.ack_mask = 0; + } + self.send_packet(key.remote_addr, &reply)? + } None => None, }; @@ -710,7 +721,7 @@ where let packets = self.refusal_packets(&mut entry.session, &refusal, true); if !packets.is_empty() { entry.ending_deadline_monotonic_millis = - Some(now_monotonic + CAPPED_ANSWER_GRACE_MILLIS); + Some(now_monotonic.saturating_add(CAPPED_ANSWER_GRACE_MILLIS)); } self.sessions.insert(key, entry); packets diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index a1954955..0e3cba66 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2022,3 +2022,37 @@ fn nothing_behind_the_capped_frame_is_served() -> Result<()> { assert!(sink.0.lock().unwrap().receipts.is_empty(), "the put behind the cap was not stored"); Ok(()) } + +/// A Ping inside the grace is answered with a Pong that acknowledges nothing: its ack fields are built from +/// what the session received, which includes the capped frame. +#[test] +fn a_pong_in_the_grace_after_the_cap_acknowledges_nothing() -> Result<()> { + let mut server = server(Default::default(), Clock::new(84_000), Sink::default(), Source::default())?; + let (mut peer, capped) = peer_at_the_cap(&mut server, 84)?; + peer.drain(); + + let ping = peer.session.create_ping(b"alive".to_vec()); + peer.send_packet(&ping)?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + let pong = peer + .drain() + .into_iter() + .find(|packet| packet.packet_type == cultnet_rs::CultNetRudpPacketType::Pong) + .expect("the ping is answered"); + assert_eq!(pong.payload, b"alive"); + assert!(!acknowledges(&pong, capped), "the Pong acknowledges the capped frame"); + assert_eq!((pong.ack, pong.ack_mask), (0, 0)); + Ok(()) +} + +/// The grace deadline saturates: at the end of the clock the answer is not given up on at once. +#[test] +fn the_grace_deadline_does_not_wrap_at_the_end_of_the_clock() -> Result<()> { + let mut server = server(Default::default(), Clock::new(u64::MAX - 10), Sink::default(), Source::default())?; + let (peer, _) = peer_at_the_cap(&mut server, 85)?; + peer.drain(); + server.maintain()?; + assert!(!peer.drain().iter().any(is_goodbye), "the deadline wrapped and the session ended before its grace"); + assert_eq!(server.session_count(), 1); + Ok(()) +} From 08cf3b34b5a27be6999ecbe5ac77d5884d6be90e Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:26:22 +0000 Subject: [PATCH 010/117] cultnet-ts test: the Odin presence publisher does not complete on the ack when the refusal arrives first Co-Authored-By: Claude Sonnet 5.5 --- .../test/idunn-runtime-authority.test.ts | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index 2b45e3b1..8b9a5629 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -279,6 +279,8 @@ async function startOdinPeer(context: TestContext, endpoint: string) { context.after(() => peer.close()); const state = { reject: false, + // How a refusal and the ack of the put reach the publisher: in one packet, or the ack first, or the refusal first. + order: "piggyback" as "piggyback" | "ack-then-refusal" | "refusal-then-ack", received: [] as Array<{ schemaVersion?: string; document: { payload: DocumentPayload } }>, responseAck: undefined as number | undefined, failure: undefined as Error | undefined, @@ -305,7 +307,15 @@ async function startOdinPeer(context: TestContext, endpoint: string) { }); assert.ok(response); state.responseAck = response.ack; - peer.send(encodeRudpPacket(response), remote.port, remote.address); + if (state.reject && state.order === "ack-then-refusal") { + peer.send(encodeRudpPacket(serverSession.createAckForReceived(packet.sequence)), remote.port, remote.address); + setTimeout(() => peer.send(encodeRudpPacket(response), remote.port, remote.address), 120); + } else if (state.reject && state.order === "refusal-then-ack") { + peer.send(encodeRudpPacket({ ...response, ack: 0, ackMask: 0 }), remote.port, remote.address); + setTimeout(() => peer.send(encodeRudpPacket(serverSession!.createAckForReceived(packet.sequence)), remote.port, remote.address), 120); + } else { + peer.send(encodeRudpPacket(response), remote.port, remote.address); + } } catch (error) { state.failure = error as Error; } @@ -332,6 +342,19 @@ test("the Odin publisher publishes the signer's document to the endpoint Expecte assert.equal(peer.failure, undefined, peer.failure?.message); }); +for (const order of ["refusal-then-ack"] as const) { + test(`the Odin publisher does not complete on the ack of a refused presence (${order})`, async (context) => { + const endpoint = expectedFacts("service").odinEndpoint!; + const peer = await startOdinPeer(context, endpoint); + peer.reject = true; + peer.order = order; + const signer = signerFor(context, "service"); + signer.reportHealth("active"); + const publisher = createIdunnRuntimePresencePublisher({ signer, endpoint }); + await assert.rejects(publisher.publish("denied presence"), /Odin rejected runtime presence: test admission denied/); + }); +} + test("the Odin publisher leaves a capability shortfall to the Rust correlation instead of refusing to publish", async (context) => { const endpoint = expectedFacts("service").odinEndpoint!; const peer = await startOdinPeer(context, endpoint); From 7766348c75b06ed112623042f3dfb356de94b0de Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:26:22 +0000 Subject: [PATCH 011/117] cultnet-ts test (FAILS, a finding): the Odin presence publisher completes on the ack when the refusal follows it Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/test/idunn-runtime-authority.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index 8b9a5629..581cf852 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -342,7 +342,7 @@ test("the Odin publisher publishes the signer's document to the endpoint Expecte assert.equal(peer.failure, undefined, peer.failure?.message); }); -for (const order of ["refusal-then-ack"] as const) { +for (const order of ["ack-then-refusal", "refusal-then-ack"] as const) { test(`the Odin publisher does not complete on the ack of a refused presence (${order})`, async (context) => { const endpoint = expectedFacts("service").odinEndpoint!; const peer = await startOdinPeer(context, endpoint); From 4f90ea815688965b4294221c4cea1453b6743946 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:27:06 +0000 Subject: [PATCH 012/117] cultmesh-ts test: publishRudpDocumentAndWaitForReceipt reports a refused put as refused, whether the ack comes before or after the refusal Co-Authored-By: Claude Sonnet 5.5 --- .../test/client-write-completion.test.ts | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 packages/cultmesh-ts/test/client-write-completion.test.ts diff --git a/packages/cultmesh-ts/test/client-write-completion.test.ts b/packages/cultmesh-ts/test/client-write-completion.test.ts new file mode 100644 index 00000000..be31e0c7 --- /dev/null +++ b/packages/cultmesh-ts/test/client-write-completion.test.ts @@ -0,0 +1,83 @@ +// Heimdall's invariant: no client may treat a reliable-transport ack as write admission. A put completes only +// on its result frame (write_not_authorized, or the success reply). Each test serves a client a peer that +// acknowledges the put and refuses it, the ack arriving before the refusal and after it. +import assert from "node:assert/strict"; +import { createSocket, type Socket } from "node:dgram"; +import test from "node:test"; +import { z } from "zod"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + CultNetRudpSession, + cultNetWriteNotAuthorizedError, + decodeRudpPacket, + defineCultNetDocumentBinding, + encodeCultNetMessageForWire, + encodeRudpPacket, +} from "@gamecult/cultnet-ts"; +import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; +import { CultMesh } from "../src/index"; + +const note = defineDocumentType({ + type: "cultmesh.completion_note", + schemaId: "cultmesh.completion_note.v0", + schema: z.object({ noteId: z.string(), text: z.string() }), + name: "noteId", +}); +const binding = defineCultNetDocumentBinding({ definition: note }); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +type Order = "ack-then-refusal" | "refusal-then-ack"; + +/** A peer that acknowledges every reliable put and refuses it with write_not_authorized, in the given order. */ +async function refusingPeer(connectionId: number, order: Order) { + const socket: Socket = createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + const session = new CultNetRudpSession({ connectionId, resendDelayMs: 50 }); + const send = (packet: Parameters[0], port: number, address: string) => + socket.send(encodeRudpPacket(packet), port, address); + socket.on("message", (wire, remote) => { + const packet = decodeRudpPacket(wire); + const now = Date.now(); + if (packet.packetType === "connect") { + send(session.acceptConnect(packet, now), remote.port, remote.address); + return; + } + const result = session.receive(packet, now); + if (result.reply) send(result.reply, remote.port, remote.address); + for (const frame of result.delivered) { + if (frame.channelId !== "schema") continue; + const ack = session.createAckForReceived(packet.sequence); + const refusal = session.sendMany( + "schema", + encodeWireValue(encodeCultNetMessageForWire(cultNetWriteNotAuthorizedError("tests:completion:a"), "cultnet.schema.v0")), + { reliable: true, ordered: true, nowMs: now }, + ); + if (order === "ack-then-refusal") { + send(ack, remote.port, remote.address); + setTimeout(() => refusal.forEach((p) => send(p, remote.port, remote.address)), 120); + } else { + // The refusal carries no acknowledgement of its own; the ack follows it. + refusal.forEach((p) => send({ ...p, ack: 0, ackMask: 0 }, remote.port, remote.address)); + setTimeout(() => send(ack, remote.port, remote.address), 120); + } + } + if (packet.reliable) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); + }); + return { endpoint: `rudp://127.0.0.1:${socket.address().port}`, close: () => socket.close() }; +} + +const orders: Order[] = ["ack-then-refusal", "refusal-then-ack"]; +let nextConnection = 0x10205000; + +for (const order of orders) { + test(`publishRudpDocumentAndWaitForReceipt reports a refused put as refused, never as success (${order})`, async () => { + const peer = await refusingPeer(nextConnection, order); + try { + await assert.rejects( + CultMesh.publishRudpDocumentAndWaitForReceipt("completion-receipt", nextConnection++, peer.endpoint, binding, "tests:completion:a", { noteId: "a", text: "x" }, binding, { resendDelayMs: 50, resendPollMs: 5, receiptTimeoutMs: 800 }), + ); + } finally { peer.close(); } + await sleep(10); + }); +} From dfd090dc557d8c08b7cf9979cf9abef3755e426d Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:27:07 +0000 Subject: [PATCH 013/117] cultmesh-ts test (FAILS, a finding): publishRudpDocumentOnce completes on the ack of a refused put Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-ts/test/client-write-completion.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/cultmesh-ts/test/client-write-completion.test.ts b/packages/cultmesh-ts/test/client-write-completion.test.ts index be31e0c7..fa3db243 100644 --- a/packages/cultmesh-ts/test/client-write-completion.test.ts +++ b/packages/cultmesh-ts/test/client-write-completion.test.ts @@ -71,6 +71,16 @@ const orders: Order[] = ["ack-then-refusal", "refusal-then-ack"]; let nextConnection = 0x10205000; for (const order of orders) { + test(`publishRudpDocumentOnce does not complete on the ack of a refused put (${order})`, async () => { + const peer = await refusingPeer(nextConnection, order); + try { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("completion-once", nextConnection++, peer.endpoint, binding, "tests:completion:a", { noteId: "a", text: "x" }, { resendDelayMs: 50, resendPollMs: 5, flushTimeoutMs: 2_000 }), + /./, + ); + } finally { peer.close(); } + }); + test(`publishRudpDocumentAndWaitForReceipt reports a refused put as refused, never as success (${order})`, async () => { const peer = await refusingPeer(nextConnection, order); try { From f53930bc135ad68f5cb0252edf5c2502ac9db0ac Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:27:48 +0000 Subject: [PATCH 014/117] cultmesh-rs test: a publish whose refusal arrives before the ack is refused Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-rs/src/lib.rs | 134 ++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index d7c2c8fd..1dec91f9 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -1451,6 +1451,126 @@ mod tests { Ok((target, stop, catalog)) } + /// A catalog that acknowledges every reliable message it receives and refuses it with a typed + /// write_not_authorized, the acknowledgement either before the refusal or after it (the second packet + /// follows the first by 100 ms). The refusal carries no acknowledgement of its own. + fn acknowledging_refusing_catalog( + refusal_first: bool, + ) -> Result<( + SocketAddr, + std::sync::Arc, + thread::JoinHandle>, + )> { + use cultnet_rs::{ + CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, + CultNetRudpSessionOptions, decode_rudp_packet, encode_rudp_packet, + }; + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(5)))?; + let target = socket.local_addr()?; + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let catalog_stop = stop.clone(); + let catalog = thread::spawn(move || -> Result<()> { + let mut session: Option = None; + let mut wire = vec![0_u8; 65_535]; + while !catalog_stop.load(std::sync::atomic::Ordering::SeqCst) { + let Ok((bytes, peer)) = socket.recv_from(&mut wire) else { + continue; + }; + let Ok(packet) = decode_rudp_packet(&wire[..bytes]) else { + continue; + }; + if packet.packet_type == CultNetRudpPacketType::Connect { + let mut accepted = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: packet.connection_id, + initial_sequence: None, + resend_delay_ms: 10_000, + max_pending_reliable_packets: Some(64), + }); + let accept = accepted.accept_connect(&packet, 0, Vec::new())?; + socket.send_to(&encode_rudp_packet(&accept)?, peer)?; + session = Some(accepted); + continue; + } + let Some(session) = session.as_mut() else { + continue; + }; + let Ok(received) = session.receive(&packet, 1) else { + continue; + }; + if !packet.reliable || received.delivered.is_empty() { + continue; + } + let ack = session.create_ack_for_received(packet.sequence); + let refusal = encode_cultnet_message_to_vec( + &cultnet_rs::CultNetMessage::Error { + error: "write_not_authorized: the catalog refused the document".into(), + code: Some(cultnet_rs::CultNetErrorCode::WriteNotAuthorized), + details: None, + }, + CultNetWireContract::CultNetSchemaV0, + )?; + let options = CultNetRudpSendOptions { + reliable: false, + ordered: false, + sequenced: false, + now_ms: 2, + reliable_expire_after_ms: None, + }; + let mut refusals = session.send_many("schema", refusal, options, Some(1200))?; + for refusal in &mut refusals { + refusal.ack = 0; + refusal.ack_mask = 0; + } + let send_refusals = |socket: &UdpSocket| -> Result<()> { + for refusal in &refusals { + socket.send_to(&encode_rudp_packet(refusal)?, peer)?; + } + Ok(()) + }; + if refusal_first { + send_refusals(&socket)?; + thread::sleep(Duration::from_millis(100)); + socket.send_to(&encode_rudp_packet(&ack)?, peer)?; + } else { + socket.send_to(&encode_rudp_packet(&ack)?, peer)?; + thread::sleep(Duration::from_millis(100)); + send_refusals(&socket)?; + } + } + Ok(()) + }); + Ok((target, stop, catalog)) + } + + fn publish_options(target: SocketAddr) -> CultMeshRudpDocumentPublishOptions { + CultMeshRudpDocumentPublishOptions { + target, + connect_timeout: Duration::from_secs(2), + flush_timeout: Duration::from_secs(3), + poll_interval: Duration::from_millis(2), + resend_delay_ms: 5, + ..CultMeshRudpDocumentPublishOptions::default() + } + } + + fn refused_note() -> cultnet_rs::CultNetMessage { + cultnet_rs::CultNetMessage::DocumentPutRaw { + message_id: "completion".into(), + document: cultnet_rs::CultNetRawDocumentRecord { + schema_id: "cultmesh.test.raw.v0".into(), + record_key: "tests:completion:a".into(), + stored_at: "2026-11-08T00:00:00Z".into(), + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: vec![0xc0], + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }, + } + } + /// A catalog's refusal text is quoted only when it is one of the fixed /// refusals a CultMesh catalog sends. Any other text, which could carry /// anything, stays out of the publisher's error. @@ -1901,4 +2021,18 @@ mod tests { assert_eq!(lease.bytes(), &[1, 2, 3]); Ok(()) } + + // Heimdall's invariant: no client treats a reliable-transport ack as write admission. The catalog here + // acknowledges the put and refuses it, the refusal arriving before or after the ack. + #[test] + fn a_publish_whose_refusal_arrives_before_the_ack_is_refused() -> Result<()> { + let (target, stop, catalog) = acknowledging_refusing_catalog(true)?; + let result = publish_cultnet_message_to_rudp_catalog(&refused_note(), publish_options(target)); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + catalog.join().expect("catalog thread should not panic")?; + let error = result.expect_err("a refused put is never reported as admitted"); + assert!(error.to_string().contains("refused the put"), "{error:#}"); + Ok(()) + } + } From 9e734447f63b970029e536d83693a264e4ff09b4 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:27:53 +0000 Subject: [PATCH 015/117] cultmesh-rs test (FAILS, findings): publish completes on the ack when the refusal follows it; the batch publish completes on the ack whatever the refusal Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-rs/src/lib.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 1dec91f9..172a5668 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -2035,4 +2035,28 @@ mod tests { Ok(()) } + + #[test] + fn a_publish_whose_ack_arrives_before_the_refusal_is_refused() -> Result<()> { + let (target, stop, catalog) = acknowledging_refusing_catalog(false)?; + let result = publish_cultnet_message_to_rudp_catalog(&refused_note(), publish_options(target)); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + catalog.join().expect("catalog thread should not panic")?; + let error = result.expect_err("a refused put is never reported as admitted"); + assert!(error.to_string().contains("refused the put"), "{error:#}"); + Ok(()) + } + + #[test] + fn a_batch_publish_is_not_complete_on_the_ack_of_refused_puts() -> Result<()> { + for refusal_first in [false, true] { + let (target, stop, catalog) = acknowledging_refusing_catalog(refusal_first)?; + let result = publish_cultnet_messages_to_rudp_catalog(&[refused_note()], publish_options(target)); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + catalog.join().expect("catalog thread should not panic")?; + assert!(result.is_err(), "refusal_first={refusal_first}: a refused put was reported as published"); + } + Ok(()) + } + } From c6da8fc54a9f904960e0a0965d0468220e878785 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:28:23 +0000 Subject: [PATCH 016/117] cultmesh-py test: a put the server refused is reported refused after its reliable ack arrived Co-Authored-By: Claude Sonnet 5.5 --- .../tests/test_client_write_completion.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 packages/cultmesh-py/tests/test_client_write_completion.py diff --git a/packages/cultmesh-py/tests/test_client_write_completion.py b/packages/cultmesh-py/tests/test_client_write_completion.py new file mode 100644 index 00000000..0f9d0c70 --- /dev/null +++ b/packages/cultmesh-py/tests/test_client_write_completion.py @@ -0,0 +1,52 @@ +"""Heimdall's invariant: no client treats a reliable-transport ack as write admission. A put completes only on +its result frame (write_not_authorized, or the success reply). + +The Python write path is a subscription over RUDP: `send` queues the put, `read_next` returns the result frame +and raises the peer's refusal. The server here refuses the put; its reliable ack of the put arrives (the test +waits for it) and the client still reports the refusal, never success. The transport's `receive` yields data +frames only, so the order of the ack against the refusal cannot change what `read_next` reports. +""" + +from __future__ import annotations + +import unittest + +from cultcache_py import define_database_entry_type +from cultmesh_py import CultMesh, create_node +from cultmesh_py.server import CultMeshLocalServer +from cultnet_py import CultNetDatabaseSubscription, CultNetPeerError, create_rudp_schema_transport + +from test_remote_write_authority import KEY_A, NOTE, SCHEMA_ID, put_of + + +class ClientWriteCompletionTests(unittest.TestCase): + def test_a_put_the_server_refused_is_reported_refused_after_its_ack_arrived(self) -> None: + node = create_node(runtime_id="completion-server") + node.database.register_document(NOTE) + server = CultMeshLocalServer(node=node, port=0, write_authority=None) + server.start() + try: + subscription = CultNetDatabaseSubscription( + host="127.0.0.1", + port=server.port, + timeout_seconds=4.0, + subscription_id="completion", + schema_ids=[SCHEMA_ID], + include_snapshot=False, + create_transport=lambda: create_rudp_schema_transport( + host="127.0.0.1", port=server.port, connection_id=server.rudp_connection_id, timeout_seconds=4.0 + ), + ) + with subscription as sub: + sub.send(put_of(KEY_A, "peer")) + sub._transport.flush_reliable(2.0) # the put's reliable ack has arrived + with self.assertRaises(CultNetPeerError) as raised: + sub.read_next() + finally: + server.stop() + self.assertEqual(raised.exception.response["code"], "write_not_authorized") + self.assertIsNone(server.node.cache.get(NOTE, KEY_A)) + + +if __name__ == "__main__": + unittest.main() From c1d24146dbc1c087de110e14173f2f3208c1fd2c Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:29:50 +0000 Subject: [PATCH 017/117] GameCult.Networking test: the RUDP schema client delivers a refused put's result frame whatever the order of the ack Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetClientWriteCompletionTests.cs | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tests/GameCult.Networking.Tests/CultNetClientWriteCompletionTests.cs diff --git a/tests/GameCult.Networking.Tests/CultNetClientWriteCompletionTests.cs b/tests/GameCult.Networking.Tests/CultNetClientWriteCompletionTests.cs new file mode 100644 index 00000000..8641a082 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetClientWriteCompletionTests.cs @@ -0,0 +1,100 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// Heimdall's invariant: no client treats a reliable-transport ack as write admission. A put completes + /// only on its result frame (write_not_authorized, or the success reply). The C# schema client has no + /// completion call: SendCultNet returns once the put is queued, and the outcome reaches the caller only + /// as the result frame, through OnCultNet. These tests serve it a peer that acknowledges the put and + /// refuses it, the ack before the refusal and after it, and require the refusal to arrive as the outcome. + /// + public class CultNetClientWriteCompletionTests + { + private const uint ConnectionId = 0x10203197; + + // Acknowledges the first reliable schema frame it receives and answers it with write_not_authorized, + // the second packet 150 ms after the first. The refusal carries no acknowledgement of its own. + private static Task RefusingPeer(Socket socket, bool refusalFirst, CancellationToken stop) + { + return Task.Run(() => + { + var session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, ResendDelayMs = 25 }); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + var answered = false; + while (!stop.IsCancellationRequested) + { + if (!socket.Poll(10_000, SelectMode.SelectRead)) + continue; + var received = socket.ReceiveFrom(buffer, ref from); + var packet = CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray()); + void Send(CultNetRudpPacket p) => socket.SendTo(CultNetRudpPacketCodec.Encode(p), from); + if (packet.PacketType == CultNetRudpPacketType.Connect) + { + Send(session.AcceptConnect(packet)); + continue; + } + var result = session.Receive(packet, 1); + if (result.Reply != null) + Send(result.Reply); + if (answered || result.Delivered.Count == 0) + continue; + answered = true; + var ack = session.CreateAckForReceived(packet.Sequence); + var refusal = session.Send("schema", CultNetSchemaMessageSerialization.Serialize(new CultNetErrorMessage + { + Error = "write_not_authorized: the catalog refused the document", + Code = "write_not_authorized" + }), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + refusal.Ack = 0; + refusal.AckMask = 0; + if (refusalFirst) + { + Send(refusal); + Thread.Sleep(150); + Send(ack); + } + else + { + Send(ack); + Thread.Sleep(150); + Send(refusal); + } + } + }); + } + + [TestCase(false, TestName = "AckThenRefusal")] + [TestCase(true, TestName = "RefusalThenAck")] + public async Task ARefusedPutIsReportedRefusedWhateverTheOrderOfItsAck(bool refusalFirst) + { + using var serverSocket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + serverSocket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + using var stop = new CancellationTokenSource(); + var peer = RefusingPeer(serverSocket, refusalFirst, stop.Token); + using var client = new RudpCultNetSchemaClient("completion", ConnectionId); + var outcomes = new BlockingCollection(); + client.OnCultNet(outcomes.Add); + client.Connect("127.0.0.1", ((IPEndPoint)serverSocket.LocalEndPoint!).Port); + var deadline = DateTime.UtcNow.AddSeconds(3); + while (!client.Connected && DateTime.UtcNow < deadline) + await Task.Delay(5); + Assert.That(client.Connected, "fixture: the client connected"); + + client.SendCultNet(new CultNetDocumentPutRawMessage { MessageId = "completion" }); + + Assert.That(outcomes.TryTake(out var refusal, 3000), Is.True, "the result frame reached the caller"); + Assert.That(refusal!.Code, Is.EqualTo("write_not_authorized")); + stop.Cancel(); + await peer; + } + } +} From 7082c9488ba0aa5cbeaf966d343a81c87c75f765 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:34:55 +0000 Subject: [PATCH 018/117] cultmesh-ts: a put completes on its result frame, not on the transport ack publishRudpDocumentOnce resolved when peer.flush returned (ack only) and read no frame, so a refused put resolved. It now rejects with CultMeshRudpWriteRefusedError (the peer's typed cultnet.error.v0: write_not_authorized, message_invalid) whether the refusal arrives before the ack, after it, or within resultWindowMs (default 500 ms) after it; a success reply ends the wait early. publishRudpDocumentAndWaitForReceipt rejects with the same typed error instead of timing out. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/src/index.ts | 44 ++++++++++++++++++- .../test/client-write-completion.test.ts | 38 +++++++++++++--- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index b77edd9f..f065dd83 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4183,7 +4183,29 @@ export interface CultMeshRudpDocumentServer { close(): void; } +/** + * A write the peer refused: the typed `cultnet.error.v0` it answered with (`write_not_authorized`, + * `message_invalid`, ...), whichever of the transport ack and the result frame came first. + */ +export class CultMeshRudpWriteRefusedError extends Error { + readonly code: string | undefined; + readonly response: Extract; + constructor(response: Extract) { + super(response.error); + this.name = "CultMeshRudpWriteRefusedError"; + this.code = response.code ?? undefined; + this.response = response; + } +} + export interface CultMeshRudpDocumentPublishOptions extends CultMeshRudpPeerOptions { + /** + * How long after the put's transport ack a refusal may still arrive. A reliable-transport ack is not + * admission; the outcome is the peer's result frame. A server that answers a put only on refusal sends no + * success frame, so the put is reported published once this window passes with no refusal; a success reply + * ends the wait early. Default 500 ms. + */ + resultWindowMs?: number; messageId?: string; sourceRuntimeId?: string; sourceAgentId?: string; @@ -6157,11 +6179,23 @@ export class CultMesh { requireNonEmpty(runtimeId, "runtimeId"); requireNonEmpty(recordKey, "recordKey"); const registry = new CultNetDocumentRegistry([binding]); + const messageId = options.messageId ?? `${runtimeId}:${binding.definition.type}:${recordKey}`; const peer = await CultMesh.createRudpPeer(runtimeId, connectionId, endpoint, options); try { + // A transport ack is not admission: the put completes on its result frame. A refusal rejects with the + // typed error whether it arrives before the ack, after it, or inside the result window. + let refusal: CultMeshRudpWriteRefusedError | undefined; + let succeeded = false; + let wake: (() => void) | undefined; + peer.on("message", (message: CultNetMessage) => { + if (message.schemaVersion === "cultnet.error.v0") refusal ??= new CultMeshRudpWriteRefusedError(message); + else if ("messageId" in message && message.messageId === messageId) succeeded = true; + else return; + wake?.(); + }); peer.send(registry.createRawDocumentPutMessage( binding, - options.messageId ?? `${runtimeId}:${binding.definition.type}:${recordKey}`, + messageId, recordKey, value, { @@ -6172,6 +6206,13 @@ export class CultMesh { }, )); await peer.flush(options.flushTimeoutMs ?? 30_000); + if (!refusal && !succeeded) { + await new Promise((resolve) => { + const timer = setTimeout(resolve, options.resultWindowMs ?? 500); + wake = () => { clearTimeout(timer); resolve(); }; + }); + } + if (refusal) throw refusal; } finally { peer.close(); } @@ -6190,6 +6231,7 @@ export class CultMesh { const receipt = new Promise>((resolve, reject) => { const timeout = setTimeout(() => { cleanup(); reject(new Error(`Timed out waiting for RUDP application receipt for ${recordKey}.`)); }, options.receiptTimeoutMs ?? 5_000); const onMessage = (message: CultNetMessage): void => { + if (message.schemaVersion === "cultnet.error.v0") { cleanup(); reject(new CultMeshRudpWriteRefusedError(message)); return; } if (message.schemaVersion !== "cultnet.document_put_raw.v0" || message.messageId !== messageId || message.document.schemaId !== receiptBinding.definition.schemaId) return; try { const decoded = decodeDocumentPayload(receiptBinding.definition, message.document.payload); diff --git a/packages/cultmesh-ts/test/client-write-completion.test.ts b/packages/cultmesh-ts/test/client-write-completion.test.ts index fa3db243..8dd122ed 100644 --- a/packages/cultmesh-ts/test/client-write-completion.test.ts +++ b/packages/cultmesh-ts/test/client-write-completion.test.ts @@ -16,7 +16,10 @@ import { encodeRudpPacket, } from "@gamecult/cultnet-ts"; import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; -import { CultMesh } from "../src/index"; +import { CultMesh, CultMeshGrantAllWrites, CultMeshRudpWriteRefusedError } from "../src/index"; + +const isTheRefusal = (error: unknown): boolean => + error instanceof CultMeshRudpWriteRefusedError && error.code === "write_not_authorized"; const note = defineDocumentType({ type: "cultmesh.completion_note", @@ -34,8 +37,12 @@ async function refusingPeer(connectionId: number, order: Order) { const socket: Socket = createSocket("udp4"); await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); const session = new CultNetRudpSession({ connectionId, resendDelayMs: 50 }); - const send = (packet: Parameters[0], port: number, address: string) => - socket.send(encodeRudpPacket(packet), port, address); + const timers = new Set(); + let closed = false; + const send = (packet: Parameters[0], port: number, address: string) => { + if (!closed) socket.send(encodeRudpPacket(packet), port, address); + }; + const later = (action: () => void) => { const timer = setTimeout(() => { timers.delete(timer); action(); }, 120); timers.add(timer); }; socket.on("message", (wire, remote) => { const packet = decodeRudpPacket(wire); const now = Date.now(); @@ -55,16 +62,16 @@ async function refusingPeer(connectionId: number, order: Order) { ); if (order === "ack-then-refusal") { send(ack, remote.port, remote.address); - setTimeout(() => refusal.forEach((p) => send(p, remote.port, remote.address)), 120); + later(() => refusal.forEach((p) => send(p, remote.port, remote.address))); } else { // The refusal carries no acknowledgement of its own; the ack follows it. refusal.forEach((p) => send({ ...p, ack: 0, ackMask: 0 }, remote.port, remote.address)); - setTimeout(() => send(ack, remote.port, remote.address), 120); + later(() => send(ack, remote.port, remote.address)); } } if (packet.reliable) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); }); - return { endpoint: `rudp://127.0.0.1:${socket.address().port}`, close: () => socket.close() }; + return { endpoint: `rudp://127.0.0.1:${socket.address().port}`, close: () => { closed = true; timers.forEach(clearTimeout); socket.close(); } }; } const orders: Order[] = ["ack-then-refusal", "refusal-then-ack"]; @@ -76,7 +83,7 @@ for (const order of orders) { try { await assert.rejects( CultMesh.publishRudpDocumentOnce("completion-once", nextConnection++, peer.endpoint, binding, "tests:completion:a", { noteId: "a", text: "x" }, { resendDelayMs: 50, resendPollMs: 5, flushTimeoutMs: 2_000 }), - /./, + isTheRefusal, ); } finally { peer.close(); } }); @@ -86,8 +93,25 @@ for (const order of orders) { try { await assert.rejects( CultMesh.publishRudpDocumentAndWaitForReceipt("completion-receipt", nextConnection++, peer.endpoint, binding, "tests:completion:a", { noteId: "a", text: "x" }, binding, { resendDelayMs: 50, resendPollMs: 5, receiptTimeoutMs: 800 }), + isTheRefusal, ); } finally { peer.close(); } await sleep(10); }); } + +test("publishRudpDocumentOnce still publishes a put the server admitted", async () => { + const connectionId = nextConnection++; + const puts: string[] = []; + const server = CultMesh.createRudpDocumentServer("completion-admit", connectionId, { + documents: new CultNetDocumentRegistry([binding]), + writeAuthority: new CultMeshGrantAllWrites(), + bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 50, resendPollMs: 5, + onDocumentPutRaw: (put) => { puts.push(put.recordKey); }, + }); + await server.start(); + try { + await CultMesh.publishRudpDocumentOnce("completion-admit-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, binding, "tests:completion:a", { noteId: "a", text: "x" }, { resendDelayMs: 50, resendPollMs: 5, resultWindowMs: 200 }); + assert.deepEqual(puts, ["tests:completion:a"]); + } finally { server.close(); } +}); From 0b326fa5eee6ccd6a27c2011778d16efc87e867d Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 11:36:23 +0000 Subject: [PATCH 019/117] WIP cultnet-ts: Odin presence publisher completes on the result frame, not the ack (passes alone; two publisher tests fail inside the parallel full run, unexplained) Co-Authored-By: Claude Sonnet 5.5 --- .../src/idunn-odin-presence-publisher.ts | 40 ++++++++++++------- packages/cultnet-ts/src/peer.ts | 5 +++ .../test/idunn-runtime-authority.test.ts | 4 +- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts b/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts index 3713363d..209a2881 100644 --- a/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts +++ b/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts @@ -107,26 +107,36 @@ async function publishDocument( const wirePayload = encode(encodeCultNetMessageForWire(message as never, "cultnet.schema.v0")); const packets = session.sendMany("schema", wirePayload, { reliable: true, ordered: true, nowMs: Date.now() }); for (const packet of packets) await sendPacket(socket, endpoint, packet); - await receiveUntil( - receiver, - session, - endpoint, - () => session.outstandingReliablePacketCount === 0, - 2000, - (frame) => { - if (frame.channelId !== "schema") return; - const response = parseCultNetMessage(decode(frame.payload)); - if (response.schemaVersion === "cultnet.error.v0") { - throw new Error(`Odin rejected runtime presence: ${response.error}`); - } - }, - ); + // The transport ack is not admission: the put completes on Odin's result frame. A refusal rejects whether + // it arrives before the ack, after it, or inside the result window; a success reply ends the wait early. + let succeeded = false; + const onFrame = (frame: { channelId: string; payload: Uint8Array }) => { + if (frame.channelId !== "schema") return; + const response = parseCultNetMessage(decode(frame.payload)); + if (response.schemaVersion === "cultnet.error.v0") throw new IdunnPresenceRefusedError(response); + succeeded = true; + }; + await receiveUntil(receiver, session, endpoint, () => session.outstandingReliablePacketCount === 0, 2000, onFrame); + if (!succeeded) await receiveUntil(receiver, session, endpoint, () => succeeded, RESULT_WINDOW_MS, onFrame, true); } finally { receiver.close(); socket.close(); } } +/** How long after the transport ack Odin's refusal may still arrive; a success reply ends the wait early. */ +const RESULT_WINDOW_MS = 500; + +/** Odin refused the presence put: the typed `cultnet.error.v0` it answered with. */ +export class IdunnPresenceRefusedError extends Error { + readonly code: string | undefined; + constructor(readonly response: { error: string; code?: string | null }) { + super(`Odin rejected runtime presence: ${response.error}`); + this.name = "IdunnPresenceRefusedError"; + this.code = response.code ?? undefined; + } +} + type PacketReceiver = { socket: dgram.Socket; next(timeoutMs: number): Promise; @@ -190,6 +200,7 @@ async function receiveUntil( predicate: (packet: CultNetRudpPacket) => boolean, timeoutMs: number, onDelivered?: (frame: { channelId: string; payload: Uint8Array; sequence: number }) => void, + returnOnTimeout = false, ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -204,6 +215,7 @@ async function receiveUntil( } for (const packet of session.dueResends(Date.now())) await sendPacket(receiver.socket, endpoint, packet); } + if (returnOnTimeout) return; throw new Error(`Timed out waiting for CultNet RUDP response after ${timeoutMs} ms.`); } diff --git a/packages/cultnet-ts/src/peer.ts b/packages/cultnet-ts/src/peer.ts index 0e54ee58..504973af 100644 --- a/packages/cultnet-ts/src/peer.ts +++ b/packages/cultnet-ts/src/peer.ts @@ -95,6 +95,10 @@ export class CultNetPeer extends EventEmitter { return this.#transport?.profile; } + /** + * Fire-and-forget: an ack or a return is not admission; read the result frame (a write is refused with + * `cultnet.error.v0`, `write_not_authorized` or `message_invalid`, as a `message` event). + */ send(message: CultNetMessage): void { const wireValue = encodeCultNetMessageForWire(message, this.#wireContract); const payload = encode(wireValue); @@ -106,6 +110,7 @@ export class CultNetPeer extends EventEmitter { this.#stream?.write(encodeFrame(payload)); } + /** Resolves when the transport has acknowledged what was sent. An ack is not admission; read the result frame. */ async flush(timeoutMs?: number): Promise { await this.#transport?.flush?.(timeoutMs); } diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index 581cf852..00d7f35b 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -309,10 +309,10 @@ async function startOdinPeer(context: TestContext, endpoint: string) { state.responseAck = response.ack; if (state.reject && state.order === "ack-then-refusal") { peer.send(encodeRudpPacket(serverSession.createAckForReceived(packet.sequence)), remote.port, remote.address); - setTimeout(() => peer.send(encodeRudpPacket(response), remote.port, remote.address), 120); + setTimeout(() => { try { peer.send(encodeRudpPacket(response), remote.port, remote.address); } catch { /* the test ended */ } }, 120); } else if (state.reject && state.order === "refusal-then-ack") { peer.send(encodeRudpPacket({ ...response, ack: 0, ackMask: 0 }), remote.port, remote.address); - setTimeout(() => peer.send(encodeRudpPacket(serverSession!.createAckForReceived(packet.sequence)), remote.port, remote.address), 120); + setTimeout(() => { try { peer.send(encodeRudpPacket(serverSession!.createAckForReceived(packet.sequence)), remote.port, remote.address); } catch { /* the test ended */ } }, 120); } else { peer.send(encodeRudpPacket(response), remote.port, remote.address); } From 531fce567d0704cca6592b22c2d919ef11cc789e Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:14:49 +0000 Subject: [PATCH 020/117] cultnet-ts test: cancel the Odin stub's delayed send when the test ends The stub's 120 ms second packet fired after peer.close() and threw ERR_SOCKET_DGRAM_NOT_RUNNING as an uncaught exception, failing whichever publisher test was running when the timer landed (load-dependent). Timers are now cleared in the test's after hook instead of swallowing the error. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/test/idunn-runtime-authority.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index 00d7f35b..2e05bbf4 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -276,7 +276,10 @@ async function startOdinPeer(context: TestContext, endpoint: string) { peer.once("error", reject); peer.bind(port, "127.0.0.1", resolve); }); - context.after(() => peer.close()); + // The second packet of a split delivery is a timer; the test's end cancels it rather than racing the closed socket. + const timers = new Set(); + const later = (send: () => void) => { timers.add(setTimeout(send, 120)); }; + context.after(() => { for (const timer of timers) clearTimeout(timer); peer.close(); }); const state = { reject: false, // How a refusal and the ack of the put reach the publisher: in one packet, or the ack first, or the refusal first. @@ -309,10 +312,10 @@ async function startOdinPeer(context: TestContext, endpoint: string) { state.responseAck = response.ack; if (state.reject && state.order === "ack-then-refusal") { peer.send(encodeRudpPacket(serverSession.createAckForReceived(packet.sequence)), remote.port, remote.address); - setTimeout(() => { try { peer.send(encodeRudpPacket(response), remote.port, remote.address); } catch { /* the test ended */ } }, 120); + later(() => peer.send(encodeRudpPacket(response), remote.port, remote.address)); } else if (state.reject && state.order === "refusal-then-ack") { peer.send(encodeRudpPacket({ ...response, ack: 0, ackMask: 0 }), remote.port, remote.address); - setTimeout(() => { try { peer.send(encodeRudpPacket(serverSession!.createAckForReceived(packet.sequence)), remote.port, remote.address); } catch { /* the test ended */ } }, 120); + later(() => peer.send(encodeRudpPacket(serverSession!.createAckForReceived(packet.sequence)), remote.port, remote.address)); } else { peer.send(encodeRudpPacket(response), remote.port, remote.address); } From 675a6494c1acdd3c288a529b68e2a7dda38575b3 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:18:40 +0000 Subject: [PATCH 021/117] cultmesh-rs: single and batch publish complete on the result frame, not the transport ack One loop now serves both: every put acknowledged, then a refusal frame (typed cultnet.error.v0) fails the call in either order against the ack, and a success frame or the end of the new result_window option (default 500 ms) completes it. The batch no longer waits on flush_reliable alone, so it cannot hide a refusal. Raw send primitives (CultNetRudpSocketTransportConnection::send/send_reliable/ flush_reliable, tcp_framed send) carry the doc line: fire-and-forget, an ack or a return is not admission, read the result frame. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-rs/src/lib.rs | 198 ++++++++++++++++----------- packages/cultnet-rs/src/rudp.rs | 6 + packages/cultnet-rs/src/transport.rs | 1 + 3 files changed, 128 insertions(+), 77 deletions(-) diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 172a5668..33cb4858 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -82,7 +82,10 @@ pub struct CultMeshRudpDocumentPublishOptions { pub target: SocketAddr, pub runtime_id: String, pub connect_timeout: Duration, + /// How long the acknowledgement of every put may take. pub flush_timeout: Duration, + /// How long after the acknowledgement a refusal may still arrive; a success frame ends the wait early. + pub result_window: Duration, pub poll_interval: Duration, pub resend_delay_ms: u64, pub source_agent_id: Option, @@ -149,6 +152,7 @@ impl Default for CultMeshRudpDocumentPublishOptions { runtime_id: "cultmesh-rudp-document-publisher".to_string(), connect_timeout: Duration::from_secs(3), flush_timeout: Duration::from_millis(300), + result_window: Duration::from_millis(500), poll_interval: Duration::from_millis(10), resend_delay_ms: 50, source_agent_id: None, @@ -882,77 +886,30 @@ fn unix_millis() -> u128 { /// Publish one raw CultNet schema message to a RUDP catalog. /// -/// Each call owns a fresh connection epoch and returns only after the catalog -/// acknowledges reliable application admission. Against a Rust CultMesh -/// catalog, which never acknowledges a message it refuses, a refusal is -/// returned as an error, naming the refusal when it is one of the fixed texts -/// such a catalog sends. A catalog that acknowledges a message on receipt and -/// refuses it afterwards (the TypeScript document server does) is reported as -/// having admitted it: the acknowledgement arrives first. +/// A put completes on the catalog's result frame, never on the transport +/// acknowledgement: an ack proves delivery, not admission. A refusal (a +/// `cultnet.error.v0` frame) returns an error in either order against the ack, +/// naming the refusal when it is one of the fixed texts a catalog sends. A +/// catalog that admits a put sends no success frame, so after the ack the call +/// waits `result_window` for a refusal and reports the put as admitted when +/// none comes (interim until the catalog sends a success frame; a success +/// frame ends the wait early). Each call owns a fresh connection epoch. pub fn publish_cultnet_message_to_rudp_catalog( message: &cultnet_rs::CultNetMessage, options: CultMeshRudpDocumentPublishOptions, ) -> Result<()> { - if options.runtime_id.trim().is_empty() { - anyhow::bail!("runtime_id must be non-empty"); - } - let mut client = connect_rudp_client( - options.target, - &options.runtime_id, - options.connect_timeout, - options.poll_interval, - options.resend_delay_ms, - "cultmesh-rudp-document-publisher", - )?; - - let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; - let receipt = client.send_reliable("schema", payload)?; - let flush_deadline = Instant::now() + options.flush_timeout; - while Instant::now() < flush_deadline { - if let Some(frame) = client.receive_once()? { - if frame.channel_id == "schema" { - if let Ok(cultnet_rs::CultNetMessage::Error { error, .. }) = - decode_cultnet_message_from_slice( - &frame.payload, - CultNetWireContract::CultNetSchemaV0, - ) - { - let _ = client.disconnect(b"document-refused".to_vec()); - anyhow::bail!( - "CultMesh RUDP catalog {} refused the put{}", - options.target, - recognised_refusal(&error) - ); - } - } - } - match client.reliable_send_status(&receipt) { - cultnet_rs::CultNetRudpReliableSendStatus::Acknowledged => { - let _ = client.disconnect(b"document-acknowledged".to_vec()); - return Ok(()); - } - cultnet_rs::CultNetRudpReliableSendStatus::Invalidated => { - anyhow::bail!("CultMesh RUDP document receipt was invalidated") - } - cultnet_rs::CultNetRudpReliableSendStatus::Pending => {} - } - client.poll_resends()?; - thread::sleep(options.poll_interval); - } - let _ = client.disconnect(b"document-unacknowledged".to_vec()); - anyhow::bail!( - "timed out waiting for reliable acknowledgement from CultMesh RUDP catalog {}", - options.target - ) + publish_cultnet_messages_to_rudp_catalog(std::slice::from_ref(message), options) } -/// Publish a batch of raw CultNet schema messages to a RUDP catalog. +/// Publish a batch of raw CultNet schema messages to a RUDP catalog on one +/// connection. /// -/// Unlike [`publish_cultnet_message_to_rudp_catalog`], which polls for -/// per-message application-level acknowledgement, this sends every message on -/// one connection and waits only for a transport-level `flush_reliable`. It -/// trades per-receipt confirmation for batch throughput; callers that need a -/// stronger delivery guarantee should use the singular publish path instead. +/// Completes under the same rule as [`publish_cultnet_message_to_rudp_catalog`]: +/// every message acknowledged, then no refusal frame within `result_window` +/// (or one success frame per message). A refusal of any message in the batch +/// fails the whole call; the batch cannot hide it. The catalog's answers do not +/// name the message they answer, so a failed batch does not say which message +/// was refused or whether the others were admitted. pub fn publish_cultnet_messages_to_rudp_catalog( messages: &[cultnet_rs::CultNetMessage], options: CultMeshRudpDocumentPublishOptions, @@ -969,11 +926,62 @@ pub fn publish_cultnet_messages_to_rudp_catalog( "cultmesh-rudp-document-publisher", )?; + let mut receipts = Vec::with_capacity(messages.len()); for message in messages { let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; - client.send("schema", payload)?; + receipts.push(client.send_reliable("schema", payload)?); + } + let ack_deadline = Instant::now() + options.flush_timeout; + let mut acknowledged_at: Option = None; + let mut successes = 0; + loop { + while let Some(frame) = client.receive_once()? { + if frame.channel_id != "schema" { + continue; + } + match decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0) { + Ok(cultnet_rs::CultNetMessage::Error { error, .. }) => { + let _ = client.disconnect(b"document-refused".to_vec()); + anyhow::bail!( + "CultMesh RUDP catalog {} refused the put{}", + options.target, + recognised_refusal(&error) + ); + } + Ok(_) => successes += 1, + Err(_) => {} + } + } + if acknowledged_at.is_none() { + let mut all_acknowledged = true; + for receipt in &receipts { + match client.reliable_send_status(receipt) { + cultnet_rs::CultNetRudpReliableSendStatus::Acknowledged => {} + cultnet_rs::CultNetRudpReliableSendStatus::Invalidated => { + anyhow::bail!("CultMesh RUDP document receipt was invalidated") + } + cultnet_rs::CultNetRudpReliableSendStatus::Pending => all_acknowledged = false, + } + } + if all_acknowledged { + acknowledged_at = Some(Instant::now()); + } else if Instant::now() >= ack_deadline { + let _ = client.disconnect(b"document-unacknowledged".to_vec()); + anyhow::bail!( + "timed out waiting for reliable acknowledgement from CultMesh RUDP catalog {}", + options.target + ) + } + } + if let Some(acknowledged_at) = acknowledged_at { + if successes >= messages.len() || acknowledged_at.elapsed() >= options.result_window { + let _ = client.disconnect(b"document-completed".to_vec()); + return Ok(()); + } + } + client.poll_resends()?; + thread::sleep(options.poll_interval); } - client.flush_reliable(options.flush_timeout) } fn connect_rudp_client( @@ -1454,8 +1462,16 @@ mod tests { /// A catalog that acknowledges every reliable message it receives and refuses it with a typed /// write_not_authorized, the acknowledgement either before the refusal or after it (the second packet /// follows the first by 100 ms). The refusal carries no acknowledgement of its own. + /// What the stub catalog sends for a put, and in which order against the transport ack. + #[derive(Clone, Copy, PartialEq, Eq)] + enum CatalogReply { + RefusalThenAck, + AckThenRefusal, + AckThenSuccess, + } + fn acknowledging_refusing_catalog( - refusal_first: bool, + reply: CatalogReply, ) -> Result<( SocketAddr, std::sync::Arc, @@ -1503,10 +1519,17 @@ mod tests { } let ack = session.create_ack_for_received(packet.sequence); let refusal = encode_cultnet_message_to_vec( - &cultnet_rs::CultNetMessage::Error { - error: "write_not_authorized: the catalog refused the document".into(), - code: Some(cultnet_rs::CultNetErrorCode::WriteNotAuthorized), - details: None, + &if reply == CatalogReply::AckThenSuccess { + cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: "completion".into(), + documents: Vec::new(), + } + } else { + cultnet_rs::CultNetMessage::Error { + error: "write_not_authorized: the catalog refused the document".into(), + code: Some(cultnet_rs::CultNetErrorCode::WriteNotAuthorized), + details: None, + } }, CultNetWireContract::CultNetSchemaV0, )?; @@ -1528,7 +1551,7 @@ mod tests { } Ok(()) }; - if refusal_first { + if reply == CatalogReply::RefusalThenAck { send_refusals(&socket)?; thread::sleep(Duration::from_millis(100)); socket.send_to(&encode_rudp_packet(&ack)?, peer)?; @@ -2026,7 +2049,7 @@ mod tests { // acknowledges the put and refuses it, the refusal arriving before or after the ack. #[test] fn a_publish_whose_refusal_arrives_before_the_ack_is_refused() -> Result<()> { - let (target, stop, catalog) = acknowledging_refusing_catalog(true)?; + let (target, stop, catalog) = acknowledging_refusing_catalog(CatalogReply::RefusalThenAck)?; let result = publish_cultnet_message_to_rudp_catalog(&refused_note(), publish_options(target)); stop.store(true, std::sync::atomic::Ordering::SeqCst); catalog.join().expect("catalog thread should not panic")?; @@ -2038,7 +2061,7 @@ mod tests { #[test] fn a_publish_whose_ack_arrives_before_the_refusal_is_refused() -> Result<()> { - let (target, stop, catalog) = acknowledging_refusing_catalog(false)?; + let (target, stop, catalog) = acknowledging_refusing_catalog(CatalogReply::AckThenRefusal)?; let result = publish_cultnet_message_to_rudp_catalog(&refused_note(), publish_options(target)); stop.store(true, std::sync::atomic::Ordering::SeqCst); catalog.join().expect("catalog thread should not panic")?; @@ -2049,14 +2072,35 @@ mod tests { #[test] fn a_batch_publish_is_not_complete_on_the_ack_of_refused_puts() -> Result<()> { - for refusal_first in [false, true] { - let (target, stop, catalog) = acknowledging_refusing_catalog(refusal_first)?; + for reply in [CatalogReply::AckThenRefusal, CatalogReply::RefusalThenAck] { + let (target, stop, catalog) = acknowledging_refusing_catalog(reply)?; let result = publish_cultnet_messages_to_rudp_catalog(&[refused_note()], publish_options(target)); stop.store(true, std::sync::atomic::Ordering::SeqCst); catalog.join().expect("catalog thread should not panic")?; - assert!(result.is_err(), "refusal_first={refusal_first}: a refused put was reported as published"); + assert!(result.is_err(), "a refused put was reported as published"); } Ok(()) } + #[test] + fn a_success_frame_completes_the_publish_without_waiting_out_the_result_window() -> Result<()> { + for batch in [false, true] { + let (target, stop, catalog) = acknowledging_refusing_catalog(CatalogReply::AckThenSuccess)?; + let options = CultMeshRudpDocumentPublishOptions { + result_window: Duration::from_secs(30), + ..publish_options(target) + }; + let started = Instant::now(); + let result = if batch { + publish_cultnet_messages_to_rudp_catalog(&[refused_note()], options) + } else { + publish_cultnet_message_to_rudp_catalog(&refused_note(), options) + }; + stop.store(true, std::sync::atomic::Ordering::SeqCst); + catalog.join().expect("catalog thread should not panic")?; + result?; + assert!(started.elapsed() < Duration::from_secs(10), "batch={batch}: waited out the window"); + } + Ok(()) + } } diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 37bf73d8..6a2aa20b 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -2198,6 +2198,7 @@ impl CultNetRudpSocketTransportConnection { self.send_packet(&packet) } + /// Fire-and-forget: an ack or a return is not admission; read the result frame. pub fn send(&mut self, channel_id: &str, payload: Vec) -> Result<()> { let options = channel_send_options( @@ -2216,6 +2217,8 @@ impl CultNetRudpSocketTransportConnection { /// Sends one frame on a reliable channel whose packets cannot expire and /// returns the transport receipt used to observe its exact acknowledgement. + /// + /// Fire-and-forget: an ack or a return is not admission; read the result frame. pub fn send_reliable( &mut self, channel_id: &str, @@ -2309,6 +2312,9 @@ impl CultNetRudpSocketTransportConnection { self.session.outstanding_reliable_packet_count() } + /// Waits until every reliable write is acknowledged by the transport. + /// + /// Fire-and-forget: an ack or a return is not admission; read the result frame. pub fn flush_reliable(&mut self, timeout: Duration) -> Result<()> { let original_timeout = self.socket.read_timeout()?; let poll_timeout = original_timeout diff --git a/packages/cultnet-rs/src/transport.rs b/packages/cultnet-rs/src/transport.rs index f2c196d4..57070dc3 100644 --- a/packages/cultnet-rs/src/transport.rs +++ b/packages/cultnet-rs/src/transport.rs @@ -252,6 +252,7 @@ impl TcpFramedTransportConnection where TStream: Write, { + /// Fire-and-forget: an ack or a return is not admission; read the result frame. pub fn send(&mut self, channel_id: &str, payload: &[u8]) -> Result<()> { if channel_id != "schema" { return Err(anyhow!( From 09e6a528b8f5c14410b062c39111c34dec8f7434 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:24:27 +0000 Subject: [PATCH 022/117] C#: SubmitDocumentAsync and the shard write forwarder complete on the result frame CultNetWriteCompletion.SendAndAwaitResultAsync registers a refusal handler before the send, then waits a result window (default 500 ms; CultMeshClientOptions. SubmitResultWindow, CultNetSchemaWriteForwarderOptions.ResultWindow). A cultnet.error.v0 inside it rejects with the new CultNetWriteRefusedException (Code: write_not_authorized, message_invalid, ...) whatever its order against the send or transport ack; a send returning no longer completes the write. The forwarder's refusal now throws on the same path as an unreachable primary, so CultNetDatabaseServer.TryForwardPutAsync/DeleteAsync no longer report a refused forward as done. Raw send primitives (C# schema client/session, Python, Kotlin) carry the doc line: fire-and-forget, an ack or a return is not admission, read the result frame. Co-Authored-By: Claude Sonnet 5.5 --- .../kotlin/org/gamecult/cultmesh/CultMesh.kt | 4 ++ packages/cultnet-py/src/cultnet_py/client.py | 2 + .../cultnet-py/src/cultnet_py/transport.py | 2 + src/GameCult.Mesh/CultMeshClient.cs | 19 +++++- src/GameCult.Mesh/CultMeshSessions.cs | 1 + .../CultNetSchemaClient.cs | 3 +- .../CultNetShardWriteForwarding.cs | 25 +++++++- .../CultNetWriteCompletion.cs | 64 +++++++++++++++++++ .../CultMeshClientTests.cs | 63 ++++++++++++++++++ .../NetworkingTests.cs | 55 ++++++++++++++++ 10 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 src/GameCult.Networking/CultNetWriteCompletion.cs diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 10cc2a58..7d052785 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -808,6 +808,7 @@ interface CultNetSchemaMessageTransport : AutoCloseable { val profile: CultNetTransportProfile val stats: CultNetTransportStats + /** Fire-and-forget: an ack or a return is not admission; read the result frame. */ fun sendSchemaMessage(message: CultNetMessage) fun receiveSchemaMessage(timeoutMs: Long = 1_000, pollIntervalMs: Long = 5): CultNetMessage? @@ -3167,13 +3168,16 @@ class CultNetRudpSocketTransportConnection( return connected } + /** Fire-and-forget: an ack or a return is not admission; read the result frame. */ fun send(channelId: String, payload: ByteArray) { session.sendMany(channelId, payload, channelSendOptions(channelId, nowMs()), maxFragmentBytes).forEach { sendPacket(it) } framesSent += 1 } + /** Fire-and-forget: an ack or a return is not admission; read the result frame. */ fun send(channelId: String, payload: String) = send(channelId, payload.toByteArray(StandardCharsets.UTF_8)) + /** Fire-and-forget: an ack or a return is not admission; read the result frame. */ fun sendSchema(payload: ByteArray) = send("schema", payload) fun sendSchema(payload: String) = sendSchema(payload.toByteArray(StandardCharsets.UTF_8)) diff --git a/packages/cultnet-py/src/cultnet_py/client.py b/packages/cultnet-py/src/cultnet_py/client.py index 28060753..35a960e3 100644 --- a/packages/cultnet-py/src/cultnet_py/client.py +++ b/packages/cultnet-py/src/cultnet_py/client.py @@ -58,6 +58,7 @@ class CultNetRawClient: create_transport: CultNetSchemaTransportFactory | None = None def send(self, message: CultNetMessage | dict[str, Any]) -> None: + """Fire-and-forget: an ack or a return is not admission; read the result frame.""" wire = message.to_wire() if isinstance(message, CultNetMessage) else message with self.open_transport() as transport: transport.send("schema", msgpack.packb(wire, use_bin_type=True)) @@ -319,6 +320,7 @@ class CultNetDatabaseSubscription: self._transport = None def send(self, message: CultNetMessage | dict[str, Any]) -> None: + """Fire-and-forget: an ack or a return is not admission; read the result frame.""" if self._transport is None: raise RuntimeError("CultNet database subscription is not open") wire = message.to_wire() if isinstance(message, CultNetMessage) else message diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 3ecfa31b..801d1313 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -1256,6 +1256,7 @@ class CultNetRudpSocketTransportConnection: self._send_packet(self.session.create_connect(_now_ms(), payload)) def send(self, channel_id: str, payload: bytes) -> None: + """Fire-and-forget: an ack or a return is not admission; read the result frame.""" packets = self.session.send_many( channel_id, payload, @@ -1567,6 +1568,7 @@ class TcpFramedTransportConnection: ) def send(self, channel_id: str, payload: bytes) -> None: + """Fire-and-forget: an ack or a return is not admission; read the result frame.""" if channel_id != "schema": raise ValueError(f"tcp_framed transport only supports the schema channel, got {channel_id!r}") before = len(payload) + 4 diff --git a/src/GameCult.Mesh/CultMeshClient.cs b/src/GameCult.Mesh/CultMeshClient.cs index 5db03ca5..5e8ef38d 100644 --- a/src/GameCult.Mesh/CultMeshClient.cs +++ b/src/GameCult.Mesh/CultMeshClient.cs @@ -44,6 +44,12 @@ namespace GameCult.Mesh /// Gets or sets the deadline for one correlated typed operation response. public TimeSpan OperationResponseTimeout { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Gets or sets how long after a submitted document the provider's refusal may still arrive. The provider + /// sends no success frame for an admitted submission, so this window bounds SubmitDocumentAsync. + /// + public TimeSpan SubmitResultWindow { get; set; } = CultNetWriteCompletion.DefaultResultWindow; } /// @@ -149,6 +155,7 @@ namespace GameCult.Mesh private readonly CultMeshSessionManager _sessions; private readonly TimeSpan _subscriptionResponseTimeout; private readonly TimeSpan _operationResponseTimeout; + private readonly TimeSpan _submitResultWindow; private readonly ConcurrentDictionary _documents = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _collections = new(StringComparer.Ordinal); private bool _disposed; @@ -191,6 +198,7 @@ namespace GameCult.Mesh options.Sessions); _subscriptionResponseTimeout = options.SubscriptionResponseTimeout; _operationResponseTimeout = options.OperationResponseTimeout; + _submitResultWindow = options.SubmitResultWindow; } /// Connects to one stable Verse/provider target using one application protocol. @@ -319,7 +327,10 @@ namespace GameCult.Mesh /// /// Submits one typed document to a provider by stable identity. Submission is uncommitted input; - /// only a provider-authored receipt or resulting state can establish acceptance. + /// only a provider-authored receipt or resulting state can establish acceptance. The task completes on + /// the provider's result frame: a refusal (write_not_authorized, message_invalid, ...) throws + /// , and a submission not refused within + /// completes. /// public async Task SubmitDocumentAsync( CultMeshSessionTarget target, @@ -351,7 +362,11 @@ namespace GameCult.Mesh var session = await ConnectAsync(target, CultMeshProtocols.Documents, cancellationToken) .ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); - session.OpenSchemaClient().SendCultNet(message); + await CultNetWriteCompletion.SendAndAwaitResultAsync( + refuse => session.OnCultNet(refuse), + () => session.SendCultNet(message), + _submitResultWindow, + cancellationToken).ConfigureAwait(false); } /// diff --git a/src/GameCult.Mesh/CultMeshSessions.cs b/src/GameCult.Mesh/CultMeshSessions.cs index 9768dfd5..9e06c196 100644 --- a/src/GameCult.Mesh/CultMeshSessions.cs +++ b/src/GameCult.Mesh/CultMeshSessions.cs @@ -352,6 +352,7 @@ namespace GameCult.Mesh internal long PhysicalGeneration => Interlocked.Read(ref _physicalGeneration); public Observable WatchState() => _states; public ICultNetSchemaClient OpenSchemaClient() => new SessionSchemaClient(this); + /// Fire-and-forget: an ack or a return is not admission; read the result frame. public void SendCultNet(T message) where T : ICultNetSchemaMessage => Channel.SendCultNet(message); public IDisposable OnCultNet(Action callback) where T : ICultNetSchemaMessage { diff --git a/src/GameCult.Networking/CultNetSchemaClient.cs b/src/GameCult.Networking/CultNetSchemaClient.cs index 5c6117aa..83865e31 100644 --- a/src/GameCult.Networking/CultNetSchemaClient.cs +++ b/src/GameCult.Networking/CultNetSchemaClient.cs @@ -26,7 +26,8 @@ namespace GameCult.Networking void Connect(string host, int port); /// - /// Sends one schema-v0 message through the transport. + /// Sends one schema-v0 message through the transport. Fire-and-forget: an ack or a return is not admission; + /// read the result frame (). /// void SendCultNet(T message) where T : ICultNetSchemaMessage; diff --git a/src/GameCult.Networking/CultNetShardWriteForwarding.cs b/src/GameCult.Networking/CultNetShardWriteForwarding.cs index 7038b0a7..0fa6c4f7 100644 --- a/src/GameCult.Networking/CultNetShardWriteForwarding.cs +++ b/src/GameCult.Networking/CultNetShardWriteForwarding.cs @@ -68,6 +68,12 @@ namespace GameCult.Networking /// rudp:// endpoints use the C# RUDP schema client and cultnet:// endpoints use LiteNetLib. /// public Func? CreateClient { get; set; } + + /// + /// Gets or sets how long after a forwarded write the primary's refusal may still arrive; a refusal fails the + /// forward with . Defaults to . + /// + public TimeSpan ResultWindow { get; set; } = CultNetWriteCompletion.DefaultResultWindow; } /// @@ -85,7 +91,10 @@ namespace GameCult.Networking _options = options ?? new CultNetSchemaWriteForwarderOptions(); } - /// + /// + /// Forwards a put and completes on the primary's result frame: a refusal throws + /// , the same failure path as an unreachable primary. + /// public async Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message) { if (message == null) throw new ArgumentNullException(nameof(message)); @@ -95,7 +104,10 @@ namespace GameCult.Networking await SendAsync(endpoint, message).ConfigureAwait(false); } - /// + /// + /// Forwards a delete and completes on the primary's result frame: a refusal throws + /// . + /// public async Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message) { if (message == null) throw new ArgumentNullException(nameof(message)); @@ -132,7 +144,14 @@ namespace GameCult.Networking using var client = CreateClient(endpoint); client.Connect(host, port); await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false); - client.SendCultNet(message); + await CultNetWriteCompletion.SendAndAwaitResultAsync( + refuse => + { + client.OnCultNet(refuse); + return null; + }, + () => client.SendCultNet(message), + _options.ResultWindow).ConfigureAwait(false); } private ICultNetSchemaClient CreateClient(string endpoint) diff --git a/src/GameCult.Networking/CultNetWriteCompletion.cs b/src/GameCult.Networking/CultNetWriteCompletion.cs new file mode 100644 index 00000000..6d5c226d --- /dev/null +++ b/src/GameCult.Networking/CultNetWriteCompletion.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace GameCult.Networking +{ + /// + /// A peer refused a write: the typed cultnet.error.v0 it answered with. + /// + public sealed class CultNetWriteRefusedException : Exception + { + /// Creates the exception from the refusal frame. + public CultNetWriteRefusedException(CultNetErrorMessage refusal) + : base(refusal?.Error ?? throw new ArgumentNullException(nameof(refusal))) + { + Code = refusal.Code; + Refusal = refusal; + } + + /// Gets the machine-readable refusal code, e.g. write_not_authorized or message_invalid. + public string? Code { get; } + + /// Gets the refusal frame as received. + public CultNetErrorMessage Refusal { get; } + } + + /// + /// Completes a write on the peer's result frame, never on a send's return or a transport acknowledgement: + /// an ack proves delivery, not admission. + /// + public static class CultNetWriteCompletion + { + /// How long after a send a refusal may still arrive when the caller names no window. + public static readonly TimeSpan DefaultResultWindow = TimeSpan.FromMilliseconds(500); + + /// + /// Sends one write and waits for the peer's answer. A refusal frame rejects with + /// whatever its order against the transport acknowledgement. A peer + /// that admits a write sends no success frame, so the end of the window is the interim bound on a refusal + /// (until peers send one); the error frame carries no message id, so any refusal inside the window is this + /// write's. + /// + /// Registers the refusal handler and returns its registration; called before the send. + /// Sends the write. + /// How long a refusal may take to arrive. + /// Cancels the wait. + public static async Task SendAndAwaitResultAsync( + Func, IDisposable?> subscribe, + Action send, + TimeSpan resultWindow, + CancellationToken cancellationToken = default) + { + if (subscribe == null) throw new ArgumentNullException(nameof(subscribe)); + if (send == null) throw new ArgumentNullException(nameof(send)); + var refused = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = subscribe(error => refused.TrySetResult(error)); + send(); + var window = Task.Delay(resultWindow, cancellationToken); + await Task.WhenAny(refused.Task, window).ConfigureAwait(false); + if (refused.Task.IsCompleted) throw new CultNetWriteRefusedException(await refused.Task.ConfigureAwait(false)); + await window.ConfigureAwait(false); + } + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshClientTests.cs b/tests/GameCult.Mesh.Tests/CultMeshClientTests.cs index dcabf6e6..9b6e3db7 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshClientTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshClientTests.cs @@ -291,6 +291,51 @@ public sealed class CultMeshClientTests submitted.Document.SourceRole.Should().Be("pilot"); } + [TestCase("write_not_authorized", 0)] + [TestCase("message_invalid", 0)] + [TestCase("write_not_authorized", 150)] + public async Task SubmitDocument_ARefusalRejectsWithTheTypedError_WhateverItsOrderAgainstTheSend(string code, int delayMs) + { + var connector = new DocumentConnector + { + RefuseSubmissionsWith = code, + RefusalDelay = TimeSpan.FromMilliseconds(delayMs) + }; + using var mesh = new CultMeshClient(new CultMeshClientOptions + { + RendezvousEndpoints = new[] { "rudp://odin:3076" }, + Discovery = new CultMeshVerseDiscoveryClientOptions { CreateClient = () => new RendezvousClient() }, + Sessions = CultMeshTestTrust.LocalSessions, + Connectors = new[] { connector } + }); + var request = new ClientTestDocument { Id = "command:1", Text = "move" }; + + var act = () => mesh.SubmitDocumentAsync(Target, request.Id, request, "pilot-one", "pilot"); + + (await act.Should().ThrowAsync()).Which.Code.Should().Be(code); + } + + [Test] + public async Task SubmitDocument_CompletesOnlyAfterTheResultWindow_NotOnTheSendReturning() + { + var connector = new DocumentConnector(); + using var mesh = new CultMeshClient(new CultMeshClientOptions + { + RendezvousEndpoints = new[] { "rudp://odin:3076" }, + Discovery = new CultMeshVerseDiscoveryClientOptions { CreateClient = () => new RendezvousClient() }, + Sessions = CultMeshTestTrust.LocalSessions, + Connectors = new[] { connector }, + SubmitResultWindow = TimeSpan.FromMilliseconds(400) + }); + var request = new ClientTestDocument { Id = "command:1", Text = "move" }; + await mesh.ConnectAsync(Target, CultMeshProtocols.Documents); + var started = DateTime.UtcNow; + + await mesh.SubmitDocumentAsync(Target, request.Id, request, "pilot-one", "pilot"); + + (DateTime.UtcNow - started).Should().BeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(380)); + } + [Test] public async Task InvokeOperation_CorrelatesProviderReceiptThroughStableIdentitySession() { @@ -496,6 +541,8 @@ public sealed class CultMeshClientTests public bool SuppressFirstSnapshot { get; set; } public bool SuppressFirstSubscriptionSnapshot { get; set; } public bool RejectOperations { get; set; } + public string? RefuseSubmissionsWith { get; set; } + public TimeSpan RefusalDelay { get; set; } public bool FailFirstOperationBeforeResponse { get; set; } public string ResponseSourceRuntimeId { get; set; } = "aetheria-daemon"; public string ConnectorId => "document-test"; @@ -515,6 +562,8 @@ public sealed class CultMeshClientTests rejectOperations: RejectOperations, failOperationBeforeResponse: FailFirstOperationBeforeResponse && Clients.Count == 0, responseSourceRuntimeId: ResponseSourceRuntimeId); + client.RefuseSubmissionsWith = RefuseSubmissionsWith; + client.RefusalDelay = RefusalDelay; Clients.Add(client); return Task.FromResult(client); } @@ -522,6 +571,9 @@ public sealed class CultMeshClientTests private sealed class DocumentClient : ICultNetSchemaClient, ICultNetSchemaClientHealth, ICultMeshVerifiedSchemaClient { + private readonly List> _errors = new(); + public string? RefuseSubmissionsWith { get; set; } + public TimeSpan RefusalDelay { get; set; } private readonly List> _snapshots = new(); private readonly List> _operationResponses = new(); private readonly CultNetDocumentRegistry _documents; @@ -558,6 +610,15 @@ public sealed class CultMeshClientTests if (message is CultNetDocumentPutRawMessage submitted) { Submitted.Add(submitted); + if (RefuseSubmissionsWith != null) + { + var refusal = new CultNetErrorMessage { Error = "not allowed", Code = RefuseSubmissionsWith }; + // A refusal that follows the send's return (and any transport ack) must still reject the submit. + Task.Delay(RefusalDelay).ContinueWith(_ => + { + foreach (var handler in _errors.ToArray()) handler(refusal); + }); + } return; } if (message is CultNetOperationRequestMessage operation) @@ -642,6 +703,8 @@ public sealed class CultMeshClientTests _snapshots.Add(response => callback((T)(object)response)); if (typeof(T) == typeof(CultNetOperationResponseMessage)) _operationResponses.Add(response => callback((T)(object)response)); + if (typeof(T) == typeof(CultNetErrorMessage)) + _errors.Add(error => callback((T)(object)error)); } public void Dispose() { } public void Fail(Exception error) => _failure.TrySetResult(error); diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 5a1f570f..2a75b5c2 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -5213,6 +5213,52 @@ namespace GameCult.Networking.Tests Assert.That(sent.ShardEpoch, Is.EqualTo(3)); } + [TestCase("write_not_authorized", 0)] + [TestCase("message_invalid", 0)] + [TestCase("write_not_authorized", 150)] + public void CultNetSchemaWriteForwarder_RejectsWithTheTypedRefusal_WhateverItsOrderAgainstTheSend(string code, int delayMs) + { + var fake = new CapturingSchemaClient(_ => null); + var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions + { + CreateClient = () => fake, + ResultWindow = TimeSpan.FromSeconds(2) + }); + var shard = new CultNetShardDescriptor( + "notes", "primary", 3, isPrimary: false, + primaryEndpoints: ["cultnet://primary.example.test:4075"]); + fake.OnSend = () => + { + var refusal = new CultNetErrorMessage { Error = "refused", Code = code }; + if (delayMs == 0) fake.Deliver(refusal); + else Task.Delay(delayMs).ContinueWith(_ => fake.Deliver(refusal)); + }; + + var put = Assert.ThrowsAsync(() => forwarder.ForwardPutAsync(shard, new CultNetDocumentPutRawMessage())); + Assert.That(put!.Code, Is.EqualTo(code)); + var delete = Assert.ThrowsAsync(() => forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { SchemaId = "note.v0", RecordKey = "note:1" })); + Assert.That(delete!.Code, Is.EqualTo(code)); + } + + [Test] + public async Task CultNetSchemaWriteForwarder_DoesNotCompleteOnTheSendReturning() + { + var fake = new CapturingSchemaClient(_ => null); + var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions + { + CreateClient = () => fake, + ResultWindow = TimeSpan.FromMilliseconds(400) + }); + var shard = new CultNetShardDescriptor( + "notes", "primary", 3, isPrimary: false, + primaryEndpoints: ["cultnet://primary.example.test:4075"]); + var started = DateTime.UtcNow; + + await forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { SchemaId = "note.v0", RecordKey = "note:1" }); + + Assert.That(DateTime.UtcNow - started, Is.GreaterThanOrEqualTo(TimeSpan.FromMilliseconds(380))); + } + [Test] public void CultMeshVerseCatalog_Finds_CompatibleTransferTargets() { @@ -7458,6 +7504,14 @@ namespace GameCult.Networking.Tests _respond = respond; } + public Action? OnSend { get; set; } + + public void Deliver(ICultNetSchemaMessage message) + { + if (!_handlers.TryGetValue(message.GetType(), out var handlers)) return; + foreach (var handler in handlers.ToArray()) handler.DynamicInvoke(message); + } + public bool Connected { get; private set; } public string? ConnectedHost { get; private set; } public int ConnectedPort { get; private set; } @@ -7474,6 +7528,7 @@ namespace GameCult.Networking.Tests where T : ICultNetSchemaMessage { SentMessages.Add(message); + OnSend?.Invoke(); var response = _respond(message); if (response == null) { From 662ae048d8621911cd0480eb862104660128bcbd Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:28:46 +0000 Subject: [PATCH 023/117] spec: cultnet write receipt (write_admitted.v0, error.v1 id echo, three-outcome clients) Specification and shared vectors only, no runtime code. A write is answered exactly once with cultnet.write_admitted.v0 or cultnet.error.v1 carrying its messageId; the new frames go only to connections whose hello lists them (today's TS decoder rejects unknown keys and versions, so "old clients ignore it" is false). Clients settle admitted | refused | unconfirmed; the result window is the pre-receipt fallback and ends unconfirmed. Wire vectors, schema-invalid cases and 16 replay scenarios in write-receipt-vectors.json. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/cultnet.error.v1.schema.json | 121 +++ .../cultnet.write-admitted.schema.json | 16 + .../interop/write-receipt-vectors.json | 727 ++++++++++++++++++ contracts/cultnet/interop/write-receipt.md | 57 ++ 4 files changed, 921 insertions(+) create mode 100644 contracts/cultnet/interop/cultnet.error.v1.schema.json create mode 100644 contracts/cultnet/interop/cultnet.write-admitted.schema.json create mode 100644 contracts/cultnet/interop/write-receipt-vectors.json create mode 100644 contracts/cultnet/interop/write-receipt.md diff --git a/contracts/cultnet/interop/cultnet.error.v1.schema.json b/contracts/cultnet/interop/cultnet.error.v1.schema.json new file mode 100644 index 00000000..255cd6bb --- /dev/null +++ b/contracts/cultnet/interop/cultnet.error.v1.schema.json @@ -0,0 +1,121 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/CultLib/contracts/cultnet/interop/cultnet.error.v1.schema.json", + "title": "CultNet Error Message v1", + "description": "cultnet.error.v0 plus messageId (write-receipt.md). Sent in place of cultnet.error.v0 only to a connection that declared the write-receipt capability. v0 is unchanged: its schema is closed (additionalProperties false), so an extra key would make a v0 peer reject the frame instead of ignoring it. Map-mode: every key is written, messageId nil when the error answers no readable message. The code enum is the v0 enum.", + "type": "object", + "required": [ + "schemaVersion", + "error", + "routingHint", + "code", + "details", + "messageId" + ], + "additionalProperties": false, + "properties": { + "schemaVersion": { + "const": "cultnet.error.v1" + }, + "error": { + "type": "string", + "minLength": 1 + }, + "routingHint": { + "description": "Optional shard routing detail for an authority failure. Null for every selection/cursor refusal code (R-N).", + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "shard", + "reason" + ], + "properties": { + "shard": { + "type": [ + "object", + "null" + ] + }, + "reason": { + "type": "string" + } + } + }, + "code": { + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", + "type": [ + "string", + "null" + ], + "enum": [ + null, + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", + "unsupported_schema_version", + "message_invalid" + ] + }, + "details": { + "description": "Code-specific structured detail (R-N). Every field is populated only for the code that defines it; the others are null, not omitted.", + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "field", + "value", + "asOf", + "current" + ], + "properties": { + "field": { + "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\". message_invalid: the request field that is missing or ill-typed (\"message\" when the whole frame cannot be read).", + "type": [ + "string", + "null" + ] + }, + "value": { + "description": "selection_invalid: the offending value, when there is one string worth naming. variant_unsupported: the id of the subscription that ended. unowned_schema: the schema id no shard owns. write_not_authorized: the key of the refused write. unsupported_schema_version: the schemaVersion the peer does not speak.", + "type": [ + "string", + "null" + ] + }, + "asOf": { + "description": "cursor_stale: the cursor's own minted asOf.", + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "current": { + "description": "cursor_stale: the answering server's current asOf.", + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + } + }, + "messageId": { + "description": "The messageId of the write (or other request) this error answers, echoed byte for byte. Null only when the frame was unreadable or carried no messageId.", + "type": [ + "string", + "null" + ], + "minLength": 1 + } + } +} diff --git a/contracts/cultnet/interop/cultnet.write-admitted.schema.json b/contracts/cultnet/interop/cultnet.write-admitted.schema.json new file mode 100644 index 00000000..42e86a2e --- /dev/null +++ b/contracts/cultnet/interop/cultnet.write-admitted.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/CultLib/contracts/cultnet/interop/cultnet.write-admitted.schema.json", + "title": "CultNet Write Admitted", + "description": "The receipt for one admitted document put or delete (write-receipt.md). Sent only after the write's commit point, only to a connection that declared the write-receipt capability, once per messageId. Map-mode: every key is written, none is nil. A receipt never carries an error, and an error.v1 never carries admission.", + "type": "object", + "required": ["schemaVersion", "messageId", "operation", "schemaId", "recordKey"], + "additionalProperties": false, + "properties": { + "schemaVersion": { "const": "cultnet.write_admitted.v0" }, + "messageId": { "description": "The messageId of the put or delete this admits, echoed byte for byte.", "type": "string", "minLength": 1 }, + "operation": { "enum": ["put", "delete"] }, + "schemaId": { "description": "The schemaId of the admitted write, echoed.", "type": "string", "minLength": 1 }, + "recordKey": { "description": "The recordKey of the admitted write, echoed.", "type": "string", "minLength": 1 } + } +} diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json new file mode 100644 index 00000000..239fc897 --- /dev/null +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -0,0 +1,727 @@ +{ + "provenance": "Spec-authored with msgpack-python 1.2.4 (str keys in schema order, use_bin_type). Not yet a C# capture: the first conformance PR for each runtime re-captures these with its own encoder and the bytes must match, as error-vectors.json does. Every schema case is checked against cultnet.write-admitted.schema.json / cultnet.error.v1.schema.json in this directory.", + "specification": "write-receipt.md", + "wire": [ + { + "name": "write_admitted_put", + "messagePackHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31", + "message": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "put-1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "name": "write_admitted_delete", + "messagePackHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A564656C2D31A96F7065726174696F6EA664656C657465A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31", + "message": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "del-1", + "operation": "delete", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "name": "error_v1_write_not_authorized_echoes_id", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93677726974655F6E6F745F617574686F72697A65643A2074686520636174616C6F6720726566757365642074686520646F63756D656E74AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A66E6F74653A32A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D32", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "write_not_authorized: the catalog refused the document", + "routingHint": null, + "code": "write_not_authorized", + "details": { + "field": "recordKey", + "value": "note:2", + "asOf": null, + "current": null + }, + "messageId": "put-2" + } + }, + { + "name": "error_v1_message_invalid_names_member_of_a_batch", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D9256D6573736167655F696E76616C69643A207265636F72644B6579206973206D697373696E67AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964B062617463682D312D6D656D6265722D33", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid: recordKey is missing", + "routingHint": null, + "code": "message_invalid", + "details": { + "field": "recordKey", + "value": null, + "asOf": null, + "current": null + }, + "messageId": "batch-1-member-3" + } + }, + { + "name": "error_v1_unreadable_frame_has_null_message_id", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D9216D6573736167655F696E76616C69643A20756E7265616461626C65206672616D65AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964C0", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid: unreadable frame", + "routingHint": null, + "code": "message_invalid", + "details": { + "field": "message", + "value": null, + "asOf": null, + "current": null + }, + "messageId": null + } + } + ], + "schemaInvalid": [ + { + "name": "write_admitted_with_an_error_key", + "messagePackHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A56572726F72A46E6F7065", + "message": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "put-1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1", + "error": "nope" + }, + "why": "admission must not carry an error" + }, + { + "name": "write_admitted_empty_message_id", + "messagePackHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A0A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31", + "message": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + }, + "why": "messageId is non-empty" + }, + { + "name": "write_admitted_unknown_operation", + "messagePackHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA6757073657274A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31", + "message": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "put-1", + "operation": "upsert", + "schemaId": "note.v0", + "recordKey": "note:1" + }, + "why": "operation is put or delete" + }, + { + "name": "write_admitted_null_record_key", + "messagePackHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579C0", + "message": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "put-1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": null + }, + "why": "map-mode keys are never nil" + }, + { + "name": "error_v1_without_message_id", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72AF6D6573736167655F696E76616C6964AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid", + "routingHint": null, + "code": "message_invalid", + "details": { + "field": "message", + "value": null, + "asOf": null, + "current": null + } + }, + "why": "messageId is required (null when unknown), never omitted" + }, + { + "name": "error_v1_empty_message_id", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72AF6D6573736167655F696E76616C6964AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A0", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid", + "routingHint": null, + "code": "message_invalid", + "details": { + "field": "message", + "value": null, + "asOf": null, + "current": null + }, + "messageId": "" + }, + "why": "null, not empty string, means unknown" + }, + { + "name": "error_v0_is_not_error_v1", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B477726974655F6E6F745F617574686F72697A6564AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A16BA461734F66C0A763757272656E74C0A96D6573736167654964A57075742D32", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": { + "field": "recordKey", + "value": "k", + "asOf": null, + "current": null + }, + "messageId": "put-2" + }, + "why": "v0 stays closed: a messageId key makes it invalid, which is why v1 exists" + } + ], + "scenarios": { + "windowMs": 500, + "semantics": "Each scenario replays events in atMs order against a client that sent each write at its send event; a tick advances the clock without a frame. expect gives the settled outcome per write id. admitted/refused/unconfirmed are three distinct results; no scenario may settle admitted without a write_admitted frame carrying the write's id.", + "cases": [ + { + "name": "receipt_admits", + "description": "A receipt with the id completes the write as admitted.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "transportAck": "w1" + }, + { + "atMs": 20, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "admitted" + } + } + }, + { + "name": "receipt_before_transport_ack", + "description": "Order against the transport ack is irrelevant.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "atMs": 9, + "transportAck": "w1" + } + ], + "expect": { + "w1": { + "outcome": "admitted" + } + } + }, + { + "name": "refusal_after_ack", + "description": "A typed refusal after the transport ack refuses; the ack was not admission.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "transportAck": "w1" + }, + { + "atMs": 120, + "frame": { + "schemaVersion": "cultnet.error.v1", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": null, + "messageId": "w1" + } + } + ], + "expect": { + "w1": { + "outcome": "refused", + "code": "write_not_authorized" + } + } + }, + { + "name": "refusal_before_ack", + "description": "A typed refusal before the transport ack refuses.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "frame": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid", + "routingHint": null, + "code": "message_invalid", + "details": null, + "messageId": "w1" + } + }, + { + "atMs": 9, + "transportAck": "w1" + } + ], + "expect": { + "w1": { + "outcome": "refused", + "code": "message_invalid" + } + } + }, + { + "name": "ack_only_window_ends_unconfirmed", + "description": "The legacy fallback: an acked write that nothing answered ends unconfirmed, never admitted.", + "server": "legacy", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "transportAck": "w1" + }, + { + "atMs": 500, + "tick": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "window_runs_from_send_not_from_ack", + "description": "A late transport ack does not extend the window.", + "server": "legacy", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 450, + "transportAck": "w1" + }, + { + "atMs": 500, + "tick": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "silent_receipt_capable_server_is_unconfirmed_too", + "description": "A server that advertised receipts and stayed silent is still only unconfirmed, and the client reports the fault.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "transportAck": "w1" + }, + { + "atMs": 5000, + "tick": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "legacy_error_with_one_write_in_flight_refuses", + "description": "A pre-receipt server answers with error.v0 and no id: attributable only because one write is unanswered.", + "server": "legacy", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 30, + "frame": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": null + } + } + ], + "expect": { + "w1": { + "outcome": "refused", + "code": "write_not_authorized" + } + } + }, + { + "name": "legacy_error_with_two_writes_in_flight_is_unattributed", + "description": "An id-less error with two unanswered writes names neither: both stay pending and end unconfirmed.", + "server": "legacy", + "writes": [ + "w1", + "w2" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 1, + "send": "w2" + }, + { + "atMs": 30, + "frame": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": null + } + }, + { + "atMs": 501, + "tick": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + }, + "w2": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "batch_names_the_refused_member", + "description": "A batch is independent writes. The refusal names its member; the others keep their own outcomes.", + "server": "receipt", + "writes": [ + "b1", + "b2", + "b3" + ], + "events": [ + { + "atMs": 0, + "send": "b1" + }, + { + "atMs": 0, + "send": "b2" + }, + { + "atMs": 0, + "send": "b3" + }, + { + "atMs": 10, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "b1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "atMs": 11, + "frame": { + "schemaVersion": "cultnet.error.v1", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": null, + "messageId": "b2" + } + }, + { + "atMs": 12, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "b3", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "b1": { + "outcome": "admitted" + }, + "b2": { + "outcome": "refused", + "code": "write_not_authorized" + }, + "b3": { + "outcome": "admitted" + } + } + }, + { + "name": "batch_with_one_member_unanswered", + "description": "A member nothing answered is unconfirmed; the batch as a whole is admitted only if every member is.", + "server": "receipt", + "writes": [ + "b1", + "b2" + ], + "events": [ + { + "atMs": 0, + "send": "b1" + }, + { + "atMs": 0, + "send": "b2" + }, + { + "atMs": 10, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "b1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "atMs": 5000, + "tick": true + } + ], + "expect": { + "b1": { + "outcome": "admitted" + }, + "b2": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "session_ends_before_an_answer", + "description": "A lost session leaves unanswered writes unconfirmed, not failed and not admitted.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "transportAck": "w1" + }, + { + "atMs": 40, + "sessionEnd": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "session_ended" + } + } + }, + { + "name": "late_refusal_does_not_rewrite_the_outcome", + "description": "An answer after the window changes nothing the caller already got; it is logged.", + "server": "legacy", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 500, + "tick": true + }, + { + "atMs": 700, + "frame": { + "schemaVersion": "cultnet.error.v1", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": null, + "messageId": "w1" + } + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "duplicate_resend_of_an_admitted_write_is_admitted_again", + "description": "The server answers a re-sent admitted messageId with the receipt again, never an error.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 20, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "atMs": 30, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "admitted" + } + } + }, + { + "name": "an_answer_for_an_unknown_id_is_ignored", + "description": "A frame naming no write in flight settles nothing.", + "server": "receipt", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 10, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "other", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "atMs": 500, + "tick": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + } + ] + } +} diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md new file mode 100644 index 00000000..4d1517bd --- /dev/null +++ b/contracts/cultnet/interop/write-receipt.md @@ -0,0 +1,57 @@ +# CultNet write receipt (specification) + +Status: specification. No runtime implements it yet; the conformance follow-ups below do. +Rules: Heimdall's ruling of 2026-11-08 (a write completes on its result frame; an ack or a return is not admission). +Files here: `cultnet.write-admitted.schema.json`, `cultnet.error.v1.schema.json`, `write-receipt-vectors.json`. They move to `contracts/cultnet/` when the first runtime implements them. + +## The problem + +A put or delete today has two answers on the wire: nothing (admitted, or the server never read it) and `cultnet.error.v0` (refused, with no id saying which write). A transport ack proves delivery. Clients that completed on the ack, or on a window with no refusal, reported refused writes as done (CultLib #55 to #58 bound this with a result window; this spec replaces the window with an answer). + +## Frames + +**`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. + +**`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. + +Why a new error version and not an extra key on v0: the v0 schema is closed (`additionalProperties: false`) and the TypeScript decoder (Ajv over the closed schemas) rejects unknown keys and unknown `schemaVersion`s, and `parseCultNetMessage` throws on an unknown version; Rust's decoder errors on an unknown version too. Unknown-key tolerance of the Rust, Python and Kotlin error decoders was not checked, so none is assumed, so a surprise key or frame breaks an old client's whole receive path. Old clients are therefore never sent either new frame. This reverses the "old clients ignore it" assumption, which is false for today's decoders. + +## Capability: opting in + +A connection is receipt-capable when its `cultnet.hello.v0` (sent before its first write; the schema channel is reliable and ordered) lists both `cultnet.write_admitted.v0` and `cultnet.error.v1` in `supportedMessageVersions`. A server MUST NOT send either new frame to any other connection, and answers it exactly as before (no receipt, `error.v0`). A receipt-capable server lists both in its own hello, so a client can tell silence from a legacy server from a receipt server's fault. A client that sends no hello gets legacy behaviour. + +## Server rules + +1. **One answer.** For every put or delete on a receipt-capable connection the server sends exactly one of `write_admitted` or `error.v1`, carrying that write's `messageId`. Never both, never two different answers. +2. **After the commit point.** `write_admitted` is sent only after the write is committed by the same rule that makes it visible to readers. Receipt of the bytes, parse, or authorisation alone is not admission. +3. **Id on every refusal.** Every refusal of a write, whatever its code (`write_not_authorized`, `message_invalid`, `unowned_schema`, `unsupported_schema_version`, shard authority errors, ...), is an `error.v1` whose `messageId` is the write's id. When the frame is readable enough to find the id, it is found; `message_invalid` for a missing or ill-typed field still carries the id. Nil only when no id can be read. +4. **Batches.** There is no batch frame; a batch is N writes, each with its own `messageId`, each answered under rules 1 to 3. A failed batch names its member by that id. A server may not collapse a batch into one answer. +5. **Idempotent resend.** A resend of the `messageId` of an already admitted write is answered with `write_admitted` again, never with an error. A resend of a refused write is answered with the same refusal. +6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server sends nothing; the client's own rule below settles it as unconfirmed. A forwarder that cannot tell admitted from refused is a fault, never an admission. +7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. + +## Client rules: three outcomes + +A write settles as exactly one of: + +- **admitted**: a `write_admitted` whose `messageId` is the write's. +- **refused**: an `error.v1` whose `messageId` is the write's, carrying its `code` and `details` as the typed error. +- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send*, default 500 ms, passed with no answer) and `session_ended` (the session ended first). Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted and decides whether to read the record back or resend (resend is safe under server rule 5). + +The result window is the *fallback* for a server that predates receipts. Against a receipt-capable server (its hello lists the frames) clients SHOULD wait longer than 500 ms (5 s suggested) and SHOULD report a silent receipt-capable server as a server fault; the outcome is still unconfirmed. + +**Legacy answers.** An `error.v0`, or an `error.v1` with nil `messageId`, names no write. It is attributed to a write only when exactly one write on the connection is unanswered; it then refuses that write. With two or more unanswered writes it settles none of them and each ends unconfirmed at its window. Late answers (after a write settled) never rewrite the outcome; log them. A frame naming a `messageId` that is not in flight settles nothing. + +A batch is the set of its members' outcomes: admitted only if every member is admitted, refused if any member is refused (naming the members), otherwise unconfirmed (naming the unanswered members). + +## Changes to what #55 to #58 shipped + +Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The conformance work changes it; until then the window-success is the interim, and callers must not read it as admission. + +## Vectors + +`write-receipt-vectors.json`: `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `scenarios` (event replays with the outcome each write must settle as; the clock is abstract, `windowMs` is 500). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. + +## Conformance follow-ups (not in this PR) + +C# server and client, Rust, TypeScript, Python, Kotlin: emit and parse the two frames; hello capability; tri-state client outcome; replace the #55 to #58 window-success; re-capture the wire vectors in each runtime. Server write paths include the shard write forwarder (rule 6) and the Odin catalog. From ab7bc771e379e98d84ea655db5c27a8802300b85 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:31:49 +0000 Subject: [PATCH 024/117] spec: write receipt per Heimdall's review (dedupe by actor+content hash, admitted-late, 5 s window, capability is framing only) - messageId unique per authenticated actor and session; dedupe key (actor, id) bound to SHA-256 of [operation, schemaId, recordKey, payload]; same id with other content is message_invalid, never the old receipt. - A late answer after the window is recorded on the write (admitted_late / refused_late); no blind resend. - A forwarder with no primary answer relays nothing; the client reports unconfirmed. - RECEIPT_RESULT_WINDOW_MS 5000 (default), LEGACY_RESULT_WINDOW_MS 500 (fallback only), DEDUPE_RETENTION_MS as vector constants. - The hello capability changes framing only and never authorizes. - Batches can commit partially; the caller owns retry. - Vectors: constants, worked content hashes, 7 dedupe cases, late-answer and window scenarios. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 339 ++++++++++++++++-- contracts/cultnet/interop/write-receipt.md | 24 +- 2 files changed, 335 insertions(+), 28 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 239fc897..93bfa4f3 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -181,8 +181,7 @@ } ], "scenarios": { - "windowMs": 500, - "semantics": "Each scenario replays events in atMs order against a client that sent each write at its send event; a tick advances the clock without a frame. expect gives the settled outcome per write id. admitted/refused/unconfirmed are three distinct results; no scenario may settle admitted without a write_admitted frame carrying the write's id.", + "semantics": "Each scenario replays events in atMs order against a client that sent each write at its send event; a tick advances the clock without a frame. 'window' names the constant in vectors.constants that applies (server receipt -> RECEIPT_RESULT_WINDOW_MS, legacy -> LEGACY_RESULT_WINDOW_MS); a tick at or after the window settles unanswered writes as unconfirmed. expect gives the outcome the caller was told per write id; recorded, when present, is the write handle's later state. admitted/refused/unconfirmed are three distinct results; no scenario may settle admitted without a write_admitted frame carrying the write's id.", "cases": [ { "name": "receipt_admits", @@ -215,7 +214,8 @@ "w1": { "outcome": "admitted" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "receipt_before_transport_ack", @@ -248,7 +248,8 @@ "w1": { "outcome": "admitted" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "refusal_after_ack", @@ -283,7 +284,8 @@ "outcome": "refused", "code": "write_not_authorized" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "refusal_before_ack", @@ -318,7 +320,8 @@ "outcome": "refused", "code": "message_invalid" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "ack_only_window_ends_unconfirmed", @@ -346,7 +349,8 @@ "outcome": "unconfirmed", "reason": "window_elapsed" } - } + }, + "window": "LEGACY_RESULT_WINDOW_MS" }, { "name": "window_runs_from_send_not_from_ack", @@ -374,7 +378,8 @@ "outcome": "unconfirmed", "reason": "window_elapsed" } - } + }, + "window": "LEGACY_RESULT_WINDOW_MS" }, { "name": "silent_receipt_capable_server_is_unconfirmed_too", @@ -402,7 +407,8 @@ "outcome": "unconfirmed", "reason": "window_elapsed" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "legacy_error_with_one_write_in_flight_refuses", @@ -432,7 +438,8 @@ "outcome": "refused", "code": "write_not_authorized" } - } + }, + "window": "LEGACY_RESULT_WINDOW_MS" }, { "name": "legacy_error_with_two_writes_in_flight_is_unattributed", @@ -475,7 +482,8 @@ "outcome": "unconfirmed", "reason": "window_elapsed" } - } + }, + "window": "LEGACY_RESULT_WINDOW_MS" }, { "name": "batch_names_the_refused_member", @@ -542,7 +550,8 @@ "b3": { "outcome": "admitted" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "batch_with_one_member_unanswered", @@ -584,7 +593,8 @@ "outcome": "unconfirmed", "reason": "window_elapsed" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "session_ends_before_an_answer", @@ -612,11 +622,12 @@ "outcome": "unconfirmed", "reason": "session_ended" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { - "name": "late_refusal_does_not_rewrite_the_outcome", - "description": "An answer after the window changes nothing the caller already got; it is logged.", + "name": "late_refusal_is_recorded_not_told", + "description": "An answer after the window does not change what the caller was told; the handle records refused_late.", "server": "legacy", "writes": [ "w1" @@ -645,9 +656,11 @@ "expect": { "w1": { "outcome": "unconfirmed", - "reason": "window_elapsed" + "reason": "window_elapsed", + "recorded": "refused_late" } - } + }, + "window": "LEGACY_RESULT_WINDOW_MS" }, { "name": "duplicate_resend_of_an_admitted_write_is_admitted_again", @@ -686,7 +699,8 @@ "w1": { "outcome": "admitted" } - } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" }, { "name": "an_answer_for_an_unknown_id_is_ignored", @@ -711,7 +725,7 @@ } }, { - "atMs": 500, + "atMs": 5000, "tick": true } ], @@ -720,7 +734,292 @@ "outcome": "unconfirmed", "reason": "window_elapsed" } + }, + "window": "RECEIPT_RESULT_WINDOW_MS" + }, + { + "name": "late_receipt_is_recorded_admitted_late", + "description": "A receipt after the window: the caller was told unconfirmed, the handle records admitted_late; a caller must consult it before resending.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5000, + "tick": true + }, + { + "atMs": 6200, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed", + "recorded": "admitted_late" + } } + }, + { + "name": "receipt_inside_the_receipt_window_is_not_late", + "description": "The receipt-capable window is 5 s: a receipt at 1.2 s admits.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 1200, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "admitted" + } + } + }, + { + "name": "legacy_window_is_not_used_against_a_receipt_server", + "description": "At 500 ms a receipt-capable server's write is still pending, not unconfirmed.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 500, + "tick": true + }, + { + "atMs": 800, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "admitted" + } + } + } + ] + }, + "constants": { + "RECEIPT_RESULT_WINDOW_MS": 5000, + "LEGACY_RESULT_WINDOW_MS": 500, + "DEDUPE_RETENTION_MS": 600000 + }, + "contentHash": { + "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", + "cases": [ + { + "name": "put_note_1_payload_a", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1", + "payloadHex": "81A26964A131", + "sha256": "7051c10071123b6ade50fbd5075e3859cae4ea70ce358a24a0b0c6980090e95d" + }, + { + "name": "put_note_1_payload_b", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1", + "payloadHex": "81A26964A132", + "sha256": "58d3fc6b858d2605a68550c9d4811ccdca6b9db7a0c3bfebf304d279257f6f9f" + }, + { + "name": "put_note_2_payload_a", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:2", + "payloadHex": "81A26964A131", + "sha256": "a248cd920e58d41bc6ccca9e108030b1d491239000047fef088122364a32b6f8" + }, + { + "name": "delete_note_1", + "operation": "delete", + "schemaId": "note.v0", + "recordKey": "note:1", + "payloadHex": null, + "sha256": "232b0f96651b28aa8ab7673a6106944a9f6a8d66852b6e21291a23ed084b8b37" + } + ] + }, + "dedupe": { + "semantics": "Replay writes in order against a receipt-capable server whose actors are transport-authenticated. expect lists the answer per write in order and how many times a write was applied to the store in total. 'invalid' means error.v1 message_invalid with details.field messageId and the write's id.", + "cases": [ + { + "name": "resend_same_content_re_admits_without_reapplying", + "writes": [ + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_a" + }, + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_a" + } + ], + "answers": [ + "admitted", + "admitted" + ], + "applied": 1 + }, + { + "name": "same_id_different_payload_is_refused_not_re_receipted", + "writes": [ + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_a" + }, + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_b" + } + ], + "answers": [ + "admitted", + "refused:message_invalid" + ], + "applied": 1 + }, + { + "name": "same_id_different_record_key_is_refused", + "writes": [ + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_a" + }, + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_2_payload_a" + } + ], + "answers": [ + "admitted", + "refused:message_invalid" + ], + "applied": 1 + }, + { + "name": "same_id_different_operation_is_refused", + "writes": [ + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_a" + }, + { + "actor": "alice", + "messageId": "m1", + "content": "delete_note_1" + } + ], + "answers": [ + "admitted", + "refused:message_invalid" + ], + "applied": 1 + }, + { + "name": "the_same_id_from_another_actor_is_an_independent_write", + "writes": [ + { + "actor": "alice", + "messageId": "m1", + "content": "put_note_1_payload_a" + }, + { + "actor": "bob", + "messageId": "m1", + "content": "put_note_1_payload_b" + } + ], + "answers": [ + "admitted", + "admitted" + ], + "applied": 2 + }, + { + "name": "resend_of_a_refused_write_is_refused_the_same_way", + "writes": [ + { + "actor": "mallory", + "messageId": "m1", + "content": "put_note_1_payload_a" + }, + { + "actor": "mallory", + "messageId": "m1", + "content": "put_note_1_payload_a" + } + ], + "answers": [ + "refused:write_not_authorized", + "refused:write_not_authorized" + ], + "applied": 0, + "note": "mallory is not authorized to write note.v0" + }, + { + "name": "the_capability_does_not_authorize", + "writes": [ + { + "actor": "mallory", + "messageId": "m2", + "content": "put_note_2_payload_a" + } + ], + "helloDeclaresReceipts": true, + "answers": [ + "refused:write_not_authorized" + ], + "applied": 0, + "note": "declaring the receipt capability changes the frame the refusal arrives in, never whether it is a refusal" } ] } diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 4d1517bd..86f9f230 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -18,7 +18,9 @@ Why a new error version and not an extra key on v0: the v0 schema is closed (`ad ## Capability: opting in -A connection is receipt-capable when its `cultnet.hello.v0` (sent before its first write; the schema channel is reliable and ordered) lists both `cultnet.write_admitted.v0` and `cultnet.error.v1` in `supportedMessageVersions`. A server MUST NOT send either new frame to any other connection, and answers it exactly as before (no receipt, `error.v0`). A receipt-capable server lists both in its own hello, so a client can tell silence from a legacy server from a receipt server's fault. A client that sends no hello gets legacy behaviour. +A connection is receipt-capable when its `cultnet.hello.v0` (sent before its first write; the schema channel is reliable and ordered) lists both `cultnet.write_admitted.v0` and `cultnet.error.v1` in `supportedMessageVersions`. A server MUST NOT send either new frame to any other connection, and answers it exactly as before (no receipt, `error.v0`). A receipt-capable server lists both in its own hello, so a client can tell a legacy server from a receipt server that has not answered yet. A client that sends no hello gets legacy behaviour. + +**The capability changes framing only; it never authorizes.** A hello authenticates nothing and grants nothing: who may write, and which keys, is decided by the server from the transport-authenticated actor exactly as without the capability. Declaring the capability never makes a refused write admitted, never widens an actor's authority, and never changes what a write does; it selects only which frame the answer arrives in. ## Server rules @@ -26,8 +28,12 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir 2. **After the commit point.** `write_admitted` is sent only after the write is committed by the same rule that makes it visible to readers. Receipt of the bytes, parse, or authorisation alone is not admission. 3. **Id on every refusal.** Every refusal of a write, whatever its code (`write_not_authorized`, `message_invalid`, `unowned_schema`, `unsupported_schema_version`, shard authority errors, ...), is an `error.v1` whose `messageId` is the write's id. When the frame is readable enough to find the id, it is found; `message_invalid` for a missing or ill-typed field still carries the id. Nil only when no id can be read. 4. **Batches.** There is no batch frame; a batch is N writes, each with its own `messageId`, each answered under rules 1 to 3. A failed batch names its member by that id. A server may not collapse a batch into one answer. -5. **Idempotent resend.** A resend of the `messageId` of an already admitted write is answered with `write_admitted` again, never with an error. A resend of a refused write is answered with the same refusal. -6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server sends nothing; the client's own rule below settles it as unconfirmed. A forwarder that cannot tell admitted from refused is a fault, never an admission. +5. **Identity and idempotent resend.** A `messageId` is unique per authenticated actor and session: the client mints it, never reuses it for different content in that session, and a server never accepts the id of one actor as the id of another. The server's dedupe key is `(actor, messageId)`, where *actor* is the transport-authenticated identity, never a name the hello or the frame claims. Alongside the key it keeps the **content hash** of the write and the answer it gave. The content hash is SHA-256 of the MessagePack array `[operation, schemaId, recordKey, payload]`, where `payload` is the raw document payload as a MessagePack `bin` for a put and nil for a delete (the vectors give worked hashes). It is retained at least `DEDUPE_RETENTION_MS` (10 minutes) and the life of the session. + - Same key, same content hash, answer was `write_admitted`: answer `write_admitted` again with the same body. The write is not applied twice. + - Same key, same content hash, answer was a refusal: the same refusal again. + - **Same key, different content hash: refuse `message_invalid` (details `field: "messageId"`) and never re-send the old receipt.** Old receipts are for the old content; showing one for new content would report an unapplied write as admitted. The original write's state is unchanged. + - Past retention a resend is a new write. +6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. ## Client rules: three outcomes @@ -36,13 +42,15 @@ A write settles as exactly one of: - **admitted**: a `write_admitted` whose `messageId` is the write's. - **refused**: an `error.v1` whose `messageId` is the write's, carrying its `code` and `details` as the typed error. -- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send*, default 500 ms, passed with no answer) and `session_ended` (the session ended first). Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted and decides whether to read the record back or resend (resend is safe under server rule 5). +- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window) and `session_ended` (the session ended first). Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content is safe against a receipt-capable server (rule 5); against a legacy server a resend may apply the write twice, so the caller reads the record back first. -The result window is the *fallback* for a server that predates receipts. Against a receipt-capable server (its hello lists the frames) clients SHOULD wait longer than 500 ms (5 s suggested) and SHOULD report a silent receipt-capable server as a server fault; the outcome is still unconfirmed. +**Result window.** Two named constants (also in the vectors): `RECEIPT_RESULT_WINDOW_MS = 5000`, the default against a receipt-capable server (its hello lists both frames); `LEGACY_RESULT_WINDOW_MS = 500`, used only as the fallback against a server whose hello does not list them (or that sent none). The window runs from the send. A receipt-capable server that has not answered within its window is reported as unconfirmed like any other; clients log it as a server that missed its window. -**Legacy answers.** An `error.v0`, or an `error.v1` with nil `messageId`, names no write. It is attributed to a write only when exactly one write on the connection is unanswered; it then refuses that write. With two or more unanswered writes it settles none of them and each ends unconfirmed at its window. Late answers (after a write settled) never rewrite the outcome; log them. A frame naming a `messageId` that is not in flight settles nothing. +**Late answers.** An answer for a write that already settled `unconfirmed` does not change what the caller was told, but the client records it on the write: a late `write_admitted` makes the write's recorded state `admitted_late`, a late refusal makes it `refused_late`. The write handle exposes that state, and a caller that got `unconfirmed` MUST consult it (and the record) before resending: it must not blind-resend a write that was admitted late. After a resend with the same id and content, rule 5 makes a duplicate harmless on a receipt-capable server. -A batch is the set of its members' outcomes: admitted only if every member is admitted, refused if any member is refused (naming the members), otherwise unconfirmed (naming the unanswered members). +**Legacy answers.** An `error.v0`, or an `error.v1` with nil `messageId`, names no write. It is attributed to a write only when exactly one write on the connection is unanswered; it then refuses that write. With two or more unanswered writes it settles none of them and each ends unconfirmed at its window. A frame naming a `messageId` that is not in flight settles nothing. + +**Batches can commit partially.** A batch is N independent writes with no atomicity: some members may be admitted while others are refused or unanswered, and nothing is rolled back. The client reports each member's outcome and a batch result is only their set: admitted if every member is admitted, refused if any member is refused (naming the members), otherwise unconfirmed (naming the unanswered members). The caller owns retry: it resends exactly the members that are not admitted, with their original ids. ## Changes to what #55 to #58 shipped @@ -50,7 +58,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `scenarios` (event replays with the outcome each write must settle as; the clock is abstract, `windowMs` is 500). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance follow-ups (not in this PR) From 9f73047c29a0154f98215d0787620a3aaef579d8 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:38:46 +0000 Subject: [PATCH 025/117] cultmesh-rs: RUDP document server answers writes with write_admitted / error.v1 (write-receipt spec) cultnet-rs gains cultnet.write_admitted.v0, cultnet.error.v1 (v0 + messageId) and write_content_hash. The server opts a session in only when its hello lists both frames (framing only; admission rules are unchanged). On an opted-in session an admitted put is answered with a receipt carrying its id after the sink accepts it, a refusal is an error.v1 naming the id, a resend of the same content is re-receipted without reaching the sink, and the same id with other content is refused message_invalid (never the old receipt). Other sessions are answered exactly as before. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 152 +++++++++++++++- .../cultmesh-rs/tests/rudp_document_server.rs | 162 ++++++++++++++++++ packages/cultnet-rs/src/contracts.rs | 111 +++++++++++- packages/cultnet-rs/tests/write_receipt.rs | 52 ++++++ 4 files changed, 462 insertions(+), 15 deletions(-) create mode 100644 packages/cultnet-rs/tests/write_receipt.rs diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 647589e0..b7d97d77 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,9 +3,9 @@ use cultnet_rs::{ CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, is_permanent_send_error, send_error_code, + encode_rudp_packet, is_permanent_send_error, send_error_code, write_content_hash, }; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; #[cfg(test)] use std::collections::BTreeSet; use std::io::ErrorKind; @@ -258,6 +258,9 @@ pub enum CultMeshRudpRejectionReason { DeleteNotSupported, /// A schema frame did not decode as a CultNet message. Carries the decoder's error text. MessageUnreadable(String), + /// A receipt-capable session sent a message id it already used for a write with other content + /// (write-receipt.md, rule 5). Nothing reached the sink; the first write's state is unchanged. + MessageIdReused, } impl std::fmt::Display for CultMeshRudpRejectionReason { @@ -266,6 +269,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { Self::SinkRefused(error) | Self::SnapshotSourceFailed(error) => f.write_str(error), Self::DeleteNotSupported => f.write_str("the server does not take deletes"), Self::MessageUnreadable(error) => write!(f, "message could not be read: {error}"), + Self::MessageIdReused => f.write_str("the message id was already used for different content"), Self::DocumentUnservable { response_bytes, max_snapshot_response_bytes, @@ -306,7 +310,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { /// The texts a refused peer is sent, one per rejection variant. They are fixed: /// a sink's or source's error text can quote what a peer sent, so it stays with /// the caller that receives the rejection and never reaches the wire. -pub(crate) const REFUSAL_TEXTS: [&str; 11] = [ +pub(crate) const REFUSAL_TEXTS: [&str; 12] = [ "the catalog refused the document", "the document can never be served", "the snapshot source failed", @@ -318,6 +322,7 @@ pub(crate) const REFUSAL_TEXTS: [&str; 11] = [ "the snapshot response could not be sent", "the server does not take deletes", "the message could not be read", + "the message id was already used for different content", ]; impl CultMeshRudpRejectionReason { @@ -335,6 +340,7 @@ impl CultMeshRudpRejectionReason { Self::ResponseSendFailed(_) => 8, Self::DeleteNotSupported => 9, Self::MessageUnreadable(_) => 10, + Self::MessageIdReused => 11, }] } } @@ -384,7 +390,23 @@ impl CultMeshRudpApplicationRejection { /// What the refused peer is sent: a refused put or delete is a typed `write_not_authorized` naming /// the record, an unreadable frame a typed `message_invalid` naming `message`; every other rejection /// is the variant's fixed text, untyped. None carries the sink's, source's or decoder's own text. - fn refusal_message(&self) -> CultNetMessage { + /// + /// A receipt-capable session is sent `cultnet.error.v1` naming the refused message's id; any other + /// session the v0 error it has always been sent. + fn refusal_message(&self, receipts: bool) -> CultNetMessage { + let refusal = self.v0_refusal_message(); + match refusal { + CultNetMessage::Error { error, code, details } if receipts => CultNetMessage::ErrorV1 { + error, + code, + details, + message_id: Some(self.message_id.clone()).filter(|id| !id.is_empty()), + }, + other => other, + } + } + + fn v0_refusal_message(&self) -> CultNetMessage { let text = self.reason.refusal_text(); match (&self.reason, &self.record_key) { ( @@ -395,6 +417,9 @@ impl CultMeshRudpApplicationRejection { (CultMeshRudpRejectionReason::MessageUnreadable(_), _) => { message_invalid_message("message", text) } + (CultMeshRudpRejectionReason::MessageIdReused, _) => { + message_invalid_message("messageId", text) + } _ => CultNetMessage::Error { error: text.into(), code: None, @@ -431,8 +456,18 @@ struct SessionEntry { /// Set once the peer's capped frame has been answered: the session ends when the answer is acknowledged, /// or at this time. ending_deadline_monotonic_millis: Option, + /// The peer's hello listed `cultnet.write_admitted.v0` and `cultnet.error.v1`: its writes are answered + /// with a receipt or an `error.v1` naming the write's id (write-receipt.md). Framing only; it grants nothing. + receipts: bool, + /// Admitted writes of a receipt-capable session: message id -> content hash, so a resend of the same + /// content is re-receipted without being applied again and the same id with other content is refused. + admitted_writes: HashMap, } +/// The most admitted writes a receipt-capable session remembers; a session at the limit refuses new writes +/// (`PayloadBudgetFull`) rather than forget an id it promised to answer idempotently. +const MAX_REMEMBERED_WRITES_PER_SESSION: usize = 1024; + /// A synchronous, multi-session CultNet RUDP raw-document server. /// /// The server owns exactly one UDP socket. Call `poll_once` from an existing @@ -687,7 +722,7 @@ where if (is_unreadable || is_delete) && run.is_some_and(|run| run < MAX_CONSECUTIVE_REFUSED_FRAMES) { - let refusal = rejection.refusal_message(); + let refusal = rejection.refusal_message(self.receipts(key)); let packets = match self.sessions.remove(&key) { Some(mut entry) => { let packets = self.refusal_packets(&mut entry.session, &refusal, false); @@ -715,7 +750,7 @@ where CultMeshRudpRejectionReason::ResponseSendFailed(_) ) { - let refusal = rejection.refusal_message(); + let refusal = rejection.refusal_message(self.receipts(key)); let packets = match self.sessions.remove(&key) { Some(mut entry) => { let packets = self.refusal_packets(&mut entry.session, &refusal, true); @@ -752,7 +787,7 @@ where rejection.reason, CultMeshRudpRejectionReason::ResponseSendFailed(_) ) { - self.end_session(key, SessionEnd::Rejected(rejection.refusal_message()))?; + self.end_session(key, SessionEnd::Rejected(rejection.refusal_message(self.receipts(key))))?; } return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } @@ -892,6 +927,8 @@ where undecodable_frames_in_a_row: 0, refused_deletes_in_a_row: 0, ending_deadline_monotonic_millis: None, + receipts: false, + admitted_writes: HashMap::new(), }, ); accept @@ -927,6 +964,39 @@ where reason, })) }; + let content_hash = match write_content_hash( + "put", + &document.schema_id, + &document.record_key, + Some(&document.payload), + ) { + Ok(hash) => hash, + Err(error) => { + return reject(CultMeshRudpRejectionReason::ResponseEncodingFailed( + format!("{error:#}"), + )); + } + }; + let remembered = self + .sessions + .get(&key) + .filter(|entry| entry.receipts) + .map(|entry| (entry.admitted_writes.get(&message_id).copied(), entry.admitted_writes.len())); + if let Some((prior, remembered_count)) = remembered { + match prior { + Some(prior) if prior == content_hash => { + self.send_write_admitted(key, now, &message_id, &document.schema_id, &document.record_key); + return Ok(None); + } + Some(_) => return reject(CultMeshRudpRejectionReason::MessageIdReused), + None if remembered_count >= MAX_REMEMBERED_WRITES_PER_SESSION => { + return reject(CultMeshRudpRejectionReason::PayloadBudgetFull); + } + None => {} + } + } + let (receipt_schema_id, receipt_record_key) = + (document.schema_id.clone(), document.record_key.clone()); // Admit only what some snapshot request can return: the smallest // response that could carry this document is the record the snapshot // source would serve for it, alone, under the shortest message id @@ -974,6 +1044,25 @@ where "{error:#}" ))); } + // Admitted: the sink holds it. Only now is a receipt owed, never before the commit point. + if let Some(entry) = self.sessions.get_mut(&key).filter(|entry| entry.receipts) { + entry.admitted_writes.insert(message_id.clone(), content_hash); + self.send_write_admitted(key, now, &message_id, &receipt_schema_id, &receipt_record_key); + } + } + CultNetMessage::Hello { + supported_message_versions, + .. + } => { + let listed = |version: &str| { + supported_message_versions + .iter() + .flatten() + .any(|candidate| candidate == version) + }; + if let Some(entry) = self.sessions.get_mut(&key) { + entry.receipts = listed("cultnet.write_admitted.v0") && listed("cultnet.error.v1"); + } } CultNetMessage::SnapshotRequest { message_id, @@ -1114,6 +1203,51 @@ where Ok(None) } + /// Sends the receipt for an admitted put. Best effort: the write is already committed, so a receipt that + /// cannot be queued or sent is dropped and the peer's own window settles the write as unconfirmed. It is + /// never answered with a refusal, which would make an admitted write look refused. + fn send_write_admitted( + &mut self, + key: CultMeshRudpSessionKey, + now: u64, + message_id: &str, + schema_id: &str, + record_key: &str, + ) { + let receipt = CultNetMessage::WriteAdmitted { + message_id: message_id.into(), + operation: "put".into(), + schema_id: schema_id.into(), + record_key: record_key.into(), + }; + let Ok(payload) = encode_cultnet_message_to_vec(&receipt, CultNetWireContract::CultNetSchemaV0) else { + return; + }; + let max_fragment_bytes = self.options.max_fragment_bytes; + let Some(entry) = self.sessions.get_mut(&key) else { + return; + }; + let Ok(packets) = entry.session.send_many( + "schema", + payload, + CultNetRudpSendOptions { + reliable: true, + ordered: true, + sequenced: false, + now_ms: now, + reliable_expire_after_ms: None, + }, + Some(max_fragment_bytes), + ) else { + return; + }; + for packet in &packets { + if !matches!(self.send_packet(key.remote_addr, packet), Ok(None)) { + break; + } + } + } + /// Ends a session before its peer is done with it, and says goodbye. It is /// the one way a session ends early: every end removes the session, sends /// what the peer is owed, then the goodbye. @@ -1155,6 +1289,10 @@ where } /// The refusal a rejected peer is sent: unreliable and unordered, or (`reliable`) queued like any response. + fn receipts(&self, key: CultMeshRudpSessionKey) -> bool { + self.sessions.get(&key).is_some_and(|entry| entry.receipts) + } + fn refusal_packets( &self, session: &mut CultNetRudpSession, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 0e3cba66..7eb6fcdc 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2056,3 +2056,165 @@ fn the_grace_deadline_does_not_wrap_at_the_end_of_the_clock() -> Result<()> { assert_eq!(server.session_count(), 1); Ok(()) } + +// Write receipts (contracts/cultnet/interop/write-receipt.md). A connection whose hello lists both new frames is +// answered with `write_admitted.v0` or `error.v1`, each carrying the write's id; any other connection is answered +// exactly as before. + +fn receipt_hello() -> CultNetMessage { + CultNetMessage::Hello { + runtime_id: "receipt-client".into(), + runtime_kind: "test".into(), + agent_id: None, + role: None, + display_name: None, + supported_document_types: None, + supported_mutation_contracts: None, + supported_message_versions: Some(vec![ + "cultnet.write_admitted.v0".into(), + "cultnet.error.v1".into(), + ]), + transport_profiles: None, + supports_schema_catalog: None, + } +} + +fn put(id: &str, key: &str, payload: Vec) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document(key, payload), + } +} + +/// Polls the server until `client` has `count` schema frames or the polling runs out; rejections are expected here. +fn frames_after( + server: &mut CultMeshRudpDocumentServer, + client: &mut CultNetRudpSocketTransportConnection, + count: usize, +) -> Result> { + let mut frames = Vec::new(); + for _ in 0..600 { + server.poll_once()?; + client.poll_resends()?; + while let Some(frame) = client.receive_once()? { + if frame.channel_id == "schema" { + frames.push(decode_cultnet_message_from_slice( + &frame.payload, + CultNetWireContract::CultNetSchemaV0, + )?); + } + } + if frames.len() >= count { + break; + } + thread::sleep(Duration::from_millis(1)); + } + Ok(frames) +} + +fn admitted(id: &str, key: &str) -> CultNetMessage { + CultNetMessage::WriteAdmitted { + message_id: id.into(), + operation: "put".into(), + schema_id: "test.raw.v1".into(), + record_key: key.into(), + } +} + +#[test] +fn an_admitted_put_on_a_receipt_connection_is_answered_with_a_receipt_carrying_its_id() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 201)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + Ok(()) +} + +#[test] +fn a_connection_without_the_capability_gets_no_receipt() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 202)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![]); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "the put was admitted all the same"); + Ok(()) +} + +#[test] +fn a_resend_of_an_admitted_write_is_re_receipted_without_applying_it_twice() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 203)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + assert_eq!( + frames_after(&mut server, &mut peer, 2)?, + vec![admitted("m1", "note:1"), admitted("m1", "note:1")] + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + Ok(()) +} + +#[test] +fn the_same_id_with_other_content_is_refused_message_invalid_never_re_receipted() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 204)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![9, 9, 9]))?; + let frames = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(frames[0], admitted("m1", "note:1")); + let CultNetMessage::ErrorV1 { code, details, message_id, .. } = &frames[1] else { + panic!("the second answer must be a refusal, got {:?}", frames[1]); + }; + assert_eq!(*code, Some(CultNetErrorCode::MessageInvalid)); + assert_eq!(message_id.as_deref(), Some("m1")); + assert_eq!(details.as_ref().and_then(|d| d.field.as_deref()), Some("messageId")); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "the changed write was not applied"); + Ok(()) +} + +#[test] +fn a_refused_put_on_a_receipt_connection_is_a_typed_error_naming_its_id() -> Result<()> { + let sink = Sink::fail_once(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 205)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m7", "note:7", vec![1]))?; + let frames = frames_after(&mut server, &mut peer, 1)?; + let Some(CultNetMessage::ErrorV1 { code, message_id, .. }) = frames.first() else { + panic!("expected error.v1, got {frames:?}"); + }; + assert_eq!(*code, Some(CultNetErrorCode::WriteNotAuthorized)); + assert_eq!(message_id.as_deref(), Some("m7")); + Ok(()) +} + +#[test] +fn a_hello_listing_only_one_of_the_two_frames_does_not_opt_in() -> Result<()> { + for listed in ["cultnet.write_admitted.v0", "cultnet.error.v1"] { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 206)?; + connect(&mut server, &mut [&mut peer])?; + let mut hello = receipt_hello(); + if let CultNetMessage::Hello { supported_message_versions, .. } = &mut hello { + *supported_message_versions = Some(vec![listed.into()]); + } + send_reliable(&mut peer, &hello)?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![], "listed only {listed}"); + } + Ok(()) +} diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index ebc32afa..cc9a9c34 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -359,6 +359,26 @@ pub enum CultNetMessage { #[serde(default, skip_serializing_if = "Option::is_none")] details: Option, }, + /// `cultnet.error.v1` (contracts/cultnet/interop/write-receipt.md): `cultnet.error.v0` plus the id of the + /// request it answers. Sent only to a connection that declared the write-receipt capability. + #[serde(skip_deserializing)] + #[serde(rename = "cultnet.error.v1", rename_all = "camelCase")] + ErrorV1 { + error: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + details: Option, + message_id: Option, + }, + /// `cultnet.write_admitted.v0`: the receipt for one admitted put or delete, sent after its commit point. + #[serde(rename = "cultnet.write_admitted.v0", rename_all = "camelCase")] + WriteAdmitted { + message_id: String, + operation: String, + schema_id: String, + record_key: String, + }, #[serde(rename = "cultnet.sample.change_name.v0", rename_all = "camelCase")] SampleChangeName { name: String }, #[serde(rename = "cultnet.sample.chat.v0", rename_all = "camelCase")] @@ -543,7 +563,8 @@ pub fn encode_cultnet_message_for_wire( | CultNetMessage::SnapshotResponseRawV1 { .. } | CultNetMessage::ContentChunkRequest { .. } | CultNetMessage::ContentChunkResponse { .. } - | CultNetMessage::Error { .. } => encode_raw_cultnet_schema_message(message), + | CultNetMessage::Error { .. } + | CultNetMessage::ErrorV1 { .. } => encode_raw_cultnet_schema_message(message), _ => Ok(cultcache_rs::decode_payload(&rmp_serde::to_vec( &serde_json::to_value(message)?, )?)?), @@ -573,7 +594,8 @@ pub fn decode_cultnet_message_from_slice( | "cultnet.snapshot_response_raw.v0" | "cultmesh.content_chunk_request.v1" | "cultmesh.content_chunk_response.v1" - | "cultnet.error.v0", + | "cultnet.error.v0" + | "cultnet.error.v1", ) => parse_raw_cultnet_schema_message(&wire_value) .with_context(|| received_version_label(&wire_value))?, _ => cultcache_rs::decode_payload::(bytes) @@ -645,6 +667,20 @@ fn validate_message(message: &CultNetMessage) -> Result<()> { require_non_empty(session, "session")?; } CultNetMessage::Error { error, code: _, details: _ } => require_non_empty(error, "error")?, + CultNetMessage::ErrorV1 { error, message_id, .. } => { + require_non_empty(error, "error")?; + if let Some(message_id) = message_id { + require_non_empty(message_id, "messageId")?; + } + } + CultNetMessage::WriteAdmitted { message_id, operation, schema_id, record_key } => { + require_non_empty(message_id, "messageId")?; + if operation != "put" && operation != "delete" { + return Err(anyhow!("operation must be put or delete")); + } + require_non_empty(schema_id, "schemaId")?; + require_non_empty(record_key, "recordKey")?; + } CultNetMessage::SampleChangeName { name } => require_non_empty(name, "name")?, CultNetMessage::SampleChat { text } => require_non_empty(text, "text")?, CultNetMessage::DocumentDelete { @@ -1021,7 +1057,7 @@ fn parse_raw_cultnet_schema_message(input: &rmpv::Value) -> Result { + "cultnet.error.v0" | "cultnet.error.v1" => { let code = get("code") .filter(|value| !value.is_nil()) .map(|value| { @@ -1032,11 +1068,19 @@ fn parse_raw_cultnet_schema_message(input: &rmpv::Value) -> Result None, + other => Some(require_legacy_string(other, "messageId")?), + }, + }); + } + Ok(CultNetMessage::Error { error, code, details }) } _ => Err(anyhow!( "Unsupported raw cultnet.schema.v0 message" @@ -1255,6 +1299,36 @@ fn encode_raw_cultnet_schema_message(message: &CultNetMessage) -> Result vec![ + (rmpv::Value::from("schemaVersion"), rmpv::Value::from("cultnet.error.v1")), + (rmpv::Value::from("error"), rmpv::Value::from(error.as_str())), + (rmpv::Value::from("routingHint"), rmpv::Value::Nil), + ( + rmpv::Value::from("code"), + code.as_ref() + .map(|code| rmpv::Value::from(code.as_wire_str())) + .unwrap_or(rmpv::Value::Nil), + ), + ( + rmpv::Value::from("details"), + details + .as_ref() + .map(encode_error_details) + .unwrap_or(rmpv::Value::Nil), + ), + ( + rmpv::Value::from("messageId"), + message_id + .as_deref() + .map(rmpv::Value::from) + .unwrap_or(rmpv::Value::Nil), + ), + ], _ => { return Err(anyhow!( "Message is not a raw cultnet.schema.v0 binary replication message" @@ -1884,3 +1958,24 @@ fn legacy_schema_descriptor(schema: &CultNetSchemaDescriptor) -> Result, +) -> Result<[u8; 32]> { + use sha2::{Digest, Sha256}; + let value = rmpv::Value::Array(vec![ + rmpv::Value::from(operation), + rmpv::Value::from(schema_id), + rmpv::Value::from(record_key), + payload.map_or(rmpv::Value::Nil, |payload| rmpv::Value::Binary(payload.to_vec())), + ]); + let mut bytes = Vec::new(); + rmpv::encode::write_value(&mut bytes, &value)?; + Ok(Sha256::digest(&bytes).into()) +} diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs new file mode 100644 index 00000000..686c3bbe --- /dev/null +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -0,0 +1,52 @@ +//! Wire parity with contracts/cultnet/interop/write-receipt-vectors.json: the content hash, and the two new frames. + +use cultnet_rs::{ + CultNetErrorCode, CultNetMessage, CultNetWireContract, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, write_content_hash, +}; + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[test] +fn the_content_hash_matches_the_spec_vectors() { + let a = [0x81, 0xA2, 0x69, 0x64, 0xA1, 0x31]; + let b = [0x81, 0xA2, 0x69, 0x64, 0xA1, 0x32]; + let cases = [ + (write_content_hash("put", "note.v0", "note:1", Some(&a)), "7051c10071123b6ade50fbd5075e3859cae4ea70ce358a24a0b0c6980090e95d"), + (write_content_hash("put", "note.v0", "note:1", Some(&b)), "58d3fc6b858d2605a68550c9d4811ccdca6b9db7a0c3bfebf304d279257f6f9f"), + (write_content_hash("put", "note.v0", "note:2", Some(&a)), "a248cd920e58d41bc6ccca9e108030b1d491239000047fef088122364a32b6f8"), + (write_content_hash("delete", "note.v0", "note:1", None), "232b0f96651b28aa8ab7673a6106944a9f6a8d66852b6e21291a23ed084b8b37"), + ]; + for (hash, expected) in cases { + assert_eq!(hex(&hash.unwrap()), expected); + } +} + +#[test] +fn the_new_frames_encode_to_the_spec_bytes() { + let admitted = CultNetMessage::WriteAdmitted { + message_id: "put-1".into(), + operation: "put".into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + }; + let bytes = encode_cultnet_message_to_vec(&admitted, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!( + decode_cultnet_message_from_slice(&bytes, CultNetWireContract::CultNetSchemaV0).unwrap(), + admitted + ); + // error.v1 with a nil id round-trips as nil, and the code enum is v0's. + let error = CultNetMessage::ErrorV1 { + error: "message_invalid: unreadable frame".into(), + code: Some(CultNetErrorCode::MessageInvalid), + details: None, + message_id: None, + }; + let bytes = encode_cultnet_message_to_vec(&error, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!( + decode_cultnet_message_from_slice(&bytes, CultNetWireContract::CultNetSchemaV0).unwrap(), + error + ); +} From 45e95984e5f72d41c76698793cdcd75865090855 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:41:14 +0000 Subject: [PATCH 026/117] WIP C# write-receipt server (not built, not tested): messages, content hash, per-peer dedupe state, receipt-aware wait Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 2 + .../CultNetSchemaMessages.cs | 70 ++++++++++++ .../CultNetWriteCompletion.cs | 59 ++++++++++ .../CultNetWriteReceipts.cs | 104 ++++++++++++++++++ 4 files changed, 235 insertions(+) create mode 100644 src/GameCult.Networking/CultNetWriteReceipts.cs diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index d098a923..cdcca912 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -53,6 +53,8 @@ namespace GameCult.Networking CultNetSchemaVersions.Verify => Deserialize(payload), CultNetSchemaVersions.LoginSuccess => Deserialize(payload), CultNetSchemaVersions.Error => Deserialize(payload), + CultNetSchemaVersions.ErrorV1 => Deserialize(payload), + CultNetSchemaVersions.WriteAdmitted => Deserialize(payload), CultNetSchemaVersions.SampleChangeName => Deserialize(payload), CultNetSchemaVersions.SampleChat => Deserialize(payload), CultNetSchemaVersions.OperationRequest => Deserialize(payload), diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 5f37f150..0a3f9adb 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -48,6 +48,14 @@ namespace GameCult.Networking /// public const string Error = "cultnet.error.v0"; /// + /// error v1 contract identifier: error v0 plus the id of the request it answers (write-receipt.md). + /// + public const string ErrorV1 = "cultnet.error.v1"; + /// + /// write admitted receipt contract identifier (write-receipt.md). + /// + public const string WriteAdmitted = "cultnet.write_admitted.v0"; + /// /// sample change name contract identifier. /// public const string SampleChangeName = "cultnet.sample.change_name.v0"; @@ -621,6 +629,68 @@ namespace GameCult.Networking }; } + /// + /// cultnet.error.v1 (contracts/cultnet/interop/write-receipt.md): the v0 error plus , + /// the id of the request it answers. Sent only to a connection that declared the write-receipt capability. + /// + [MessagePackObject] + public class CultNetErrorV1Message : ICultNetSchemaMessage + { + /// Gets or sets the schema version. + [Key("schemaVersion")] public string SchemaVersion { get; set; } = CultNetSchemaVersions.ErrorV1; + /// Gets or sets the error. + [Key("error")] public string Error { get; set; } = string.Empty; + /// Gets or sets optional shard routing details for authority failures. + [Key("routingHint")] public CultNetShardRoutingHint? RoutingHint { get; set; } + /// Gets or sets the machine-readable refusal code (the v0 enum). + [Key("code")] public string? Code { get; set; } + /// Gets or sets the code-specific structured detail. + [Key("details")] public CultNetErrorDetails? Details { get; set; } + /// Gets or sets the id of the request this answers; null only when no id could be read. + [Key("messageId")] public string? MessageId { get; set; } + + /// Builds the v1 form of answering . + public static CultNetErrorV1Message From(CultNetErrorMessage error, string? messageId) + { + if (error == null) throw new ArgumentNullException(nameof(error)); + return new CultNetErrorV1Message + { + Error = error.Error, + RoutingHint = error.RoutingHint, + Code = error.Code, + Details = error.Details, + MessageId = string.IsNullOrEmpty(messageId) ? null : messageId + }; + } + + /// Gets the v0 form, dropping the id. + public CultNetErrorMessage ToV0() => new CultNetErrorMessage + { + Error = Error, + RoutingHint = RoutingHint, + Code = Code, + Details = Details + }; + } + + /// + /// cultnet.write_admitted.v0: the receipt for one admitted put or delete, sent after its commit point. + /// + [MessagePackObject] + public class CultNetWriteAdmittedMessage : ICultNetSchemaMessage + { + /// Gets or sets the schema version. + [Key("schemaVersion")] public string SchemaVersion { get; set; } = CultNetSchemaVersions.WriteAdmitted; + /// Gets or sets the messageId of the admitted write, echoed. + [Key("messageId")] public string MessageId { get; set; } = string.Empty; + /// Gets or sets put or delete. + [Key("operation")] public string Operation { get; set; } = string.Empty; + /// Gets or sets the schemaId of the admitted write, echoed. + [Key("schemaId")] public string SchemaId { get; set; } = string.Empty; + /// Gets or sets the recordKey of the admitted write, echoed. + [Key("recordKey")] public string RecordKey { get; set; } = string.Empty; + } + /// /// Code-specific structured detail carried on (R-N). Fields are /// populated only for the code that defines them; every field is otherwise omitted (nil on the wire). diff --git a/src/GameCult.Networking/CultNetWriteCompletion.cs b/src/GameCult.Networking/CultNetWriteCompletion.cs index 6d5c226d..be8a4632 100644 --- a/src/GameCult.Networking/CultNetWriteCompletion.cs +++ b/src/GameCult.Networking/CultNetWriteCompletion.cs @@ -24,6 +24,22 @@ namespace GameCult.Networking public CultNetErrorMessage Refusal { get; } } + /// + /// A write nothing answered within its window: neither admitted nor refused (write-receipt.md, "unconfirmed"). + /// + public sealed class CultNetWriteUnconfirmedException : Exception + { + /// Creates the exception for the write . + public CultNetWriteUnconfirmedException(string messageId) + : base($"No admission or refusal arrived for write '{messageId}'.") + { + MessageId = messageId; + } + + /// Gets the id of the unconfirmed write. + public string MessageId { get; } + } + /// /// Completes a write on the peer's result frame, never on a send's return or a transport acknowledgement: /// an ack proves delivery, not admission. @@ -33,6 +49,49 @@ namespace GameCult.Networking /// How long after a send a refusal may still arrive when the caller names no window. public static readonly TimeSpan DefaultResultWindow = TimeSpan.FromMilliseconds(500); + /// How long a receipt-capable peer has to answer a write (write-receipt.md, RECEIPT_RESULT_WINDOW_MS). + public static readonly TimeSpan ReceiptResultWindow = TimeSpan.FromMilliseconds(5000); + + /// + /// Sends one write to a receipt-capable peer and waits for its answer: a + /// write_admitted naming returns, an error.v1 naming it throws + /// , and an id-less error (v0, or v1 with no id) is attributed to this write + /// because it is the only one in flight on the connection. Silence throws . + /// + public static async Task SendAndAwaitReceiptAsync( + Func, Action, Action, IDisposable?> subscribe, + Action send, + string messageId, + TimeSpan resultWindow, + CancellationToken cancellationToken = default) + { + if (subscribe == null) throw new ArgumentNullException(nameof(subscribe)); + if (send == null) throw new ArgumentNullException(nameof(send)); + var answer = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = subscribe( + admitted => + { + if (string.Equals(admitted.MessageId, messageId, StringComparison.Ordinal)) answer.TrySetResult(null); + }, + error => answer.TrySetResult(error), + error => + { + if (error.MessageId == null || string.Equals(error.MessageId, messageId, StringComparison.Ordinal)) + answer.TrySetResult(error.ToV0()); + }); + send(); + var window = Task.Delay(resultWindow, cancellationToken); + await Task.WhenAny(answer.Task, window).ConfigureAwait(false); + if (!answer.Task.IsCompleted) + { + await window.ConfigureAwait(false); + throw new CultNetWriteUnconfirmedException(messageId); + } + + var refusal = await answer.Task.ConfigureAwait(false); + if (refusal != null) throw new CultNetWriteRefusedException(refusal); + } + /// /// Sends one write and waits for the peer's answer. A refusal frame rejects with /// whatever its order against the transport acknowledgement. A peer diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs new file mode 100644 index 00000000..2cc65db8 --- /dev/null +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using MessagePack; + +namespace GameCult.Networking +{ + /// + /// The content hash a write-receipt server binds to a (peer, messageId) dedupe key + /// (contracts/cultnet/interop/write-receipt.md, rule 5). + /// + public static class CultNetWriteContentHash + { + /// + /// SHA-256 of the MessagePack array [operation, schemaId, recordKey, payload], payload a bin + /// for a put and nil for a delete. + /// + public static byte[] Compute(string operation, string schemaId, string recordKey, byte[]? payload) + { + var buffer = new System.Buffers.ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(4); + writer.Write(operation); + writer.Write(schemaId); + writer.Write(recordKey); + if (payload == null) writer.WriteNil(); + else writer.Write(payload); + writer.Flush(); + return SHA256.HashData(buffer.WrittenSpan); + } + } + + /// + /// Whether a peer's hello opted in to write receipts: it lists both cultnet.write_admitted.v0 and + /// cultnet.error.v1. The capability changes framing only; it never authorizes. + /// + internal static class CultNetWriteReceiptCapability + { + public static bool Declares(CultNetHelloMessage hello) + { + var versions = hello.SupportedMessageVersions; + return versions != null + && Array.IndexOf(versions, CultNetSchemaVersions.WriteAdmitted) >= 0 + && Array.IndexOf(versions, CultNetSchemaVersions.ErrorV1) >= 0; + } + } + + /// + /// One receipt-capable peer's writes: message id to content hash, so a resend of the same content is re-receipted + /// without being committed again and the same id with other content is refused. + /// + internal sealed class CultNetWriteReceiptSession + { + /// The most writes a peer is remembered for; a peer at the limit has new writes refused. + internal const int MaxRememberedWrites = 1024; + + private readonly object _gate = new(); + private readonly Dictionary _writes = new(StringComparer.Ordinal); + + private sealed class Entry + { + public Entry(byte[] hash) { Hash = hash; } + public byte[] Hash { get; } + public bool Admitted { get; set; } + } + + public enum Begin { New, AdmittedBefore, InFlight, DifferentContent, Full } + + /// Claims for a write with this content, or says what already holds it. + public Begin TryBegin(string messageId, byte[] hash) + { + lock (_gate) + { + if (_writes.TryGetValue(messageId, out var entry)) + { + if (!CryptographicOperations.FixedTimeEquals(entry.Hash, hash)) return Begin.DifferentContent; + return entry.Admitted ? Begin.AdmittedBefore : Begin.InFlight; + } + + if (_writes.Count >= MaxRememberedWrites) return Begin.Full; + _writes[messageId] = new Entry(hash); + return Begin.New; + } + } + + /// The write committed: remember it so a resend is re-receipted. + public void Admitted(string messageId) + { + lock (_gate) + { + if (_writes.TryGetValue(messageId, out var entry)) entry.Admitted = true; + } + } + + /// The write was refused or unanswered: forget it so a corrected or repeated attempt is judged afresh. + public void Abandoned(string messageId) + { + lock (_gate) + { + _writes.Remove(messageId); + } + } + } +} From 8bd31a5d4aa5ec14426466753a7926e913055fac Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:56:17 +0000 Subject: [PATCH 027/117] cultmesh-rs: receipt server follows Heimdall's review of #60 A refused put, a full dedupe table and a reused id refuse the write, not the session; only a protocol violation ends one. A frame that does not decode but holds a messageId is refused with that id. The server answers a receipt-capable hello with its own, listing both frames. The sink's authorization (new authorize_raw_document, default admit) runs before the session's remembered writes are consulted. A second hello does not change the capability: it is set once, by the first hello or the first write. A receipt that cannot be queued, and a write the payload budget would drop silently, end the session instead of leaving the write unanswered. Refusals to a receipt-capable session are queued reliably. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 229 ++++++++++++------ .../cultmesh-rs/tests/rudp_document_server.rs | 228 ++++++++++++++++- packages/cultnet-rs/src/contracts.rs | 14 ++ packages/cultnet-rs/tests/write_receipt.rs | 2 +- 4 files changed, 397 insertions(+), 76 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index b7d97d77..173ebf3b 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,7 +3,7 @@ use cultnet_rs::{ CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, is_permanent_send_error, send_error_code, write_content_hash, + encode_rudp_packet, is_permanent_send_error, readable_message_id, send_error_code, write_content_hash, }; use std::collections::{BTreeMap, HashMap}; #[cfg(test)] @@ -66,6 +66,14 @@ pub struct CultMeshRudpSnapshotQuery { /// Caller-owned admission/persistence port for received raw documents. pub trait CultMeshRudpRawDocumentSink { + /// Whether this put may be written, decided without writing it. A receipt-capable session's resend or + /// reused id is checked against it before the session's remembered writes are consulted, so a peer that + /// may not write is refused as such and never shown what the session remembers (write-receipt.md). + /// The default admits everything and leaves the decision to `accept_raw_document`. + fn authorize_raw_document(&mut self, _receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + Ok(()) + } + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()>; } @@ -459,6 +467,9 @@ struct SessionEntry { /// The peer's hello listed `cultnet.write_admitted.v0` and `cultnet.error.v1`: its writes are answered /// with a receipt or an `error.v1` naming the write's id (write-receipt.md). Framing only; it grants nothing. receipts: bool, + /// The capability is decided once, by the session's first hello or first write, whichever comes first; + /// a later hello changes nothing. + capability_settled: bool, /// Admitted writes of a receipt-capable session: message id -> content hash, so a resend of the same /// content is re-receipted without being applied again and the same id with other content is refused. admitted_writes: HashMap, @@ -466,6 +477,9 @@ struct SessionEntry { /// The most admitted writes a receipt-capable session remembers; a session at the limit refuses new writes /// (`PayloadBudgetFull`) rather than forget an id it promised to answer idempotently. +const WRITE_ADMITTED_VERSION: &str = "cultnet.write_admitted.v0"; +const ERROR_V1_VERSION: &str = "cultnet.error.v1"; + const MAX_REMEMBERED_WRITES_PER_SESSION: usize = 1024; /// A synchronous, multi-session CultNet RUDP raw-document server. @@ -617,6 +631,11 @@ where 0 }; if !self.payload_budget_allows(key, payload_charge) { + // A receipt-capable session's dropped write would go unanswered; ending the session answers it + // (the peer's writes settle as session_ended). Any other session is dropped from silently, as before. + if self.receipts(key) { + self.end_session(key, SessionEnd::Refused)?; + } return Ok(CultMeshRudpPollOutcome::Handled); } @@ -711,7 +730,7 @@ where Err(error) => Some(CultMeshRudpApplicationRejection { session: key, operation: CultMeshRudpApplicationOperation::Unreadable, - message_id: String::new(), + message_id: readable_message_id(&frame.payload).unwrap_or_default(), record_key: None, reason: CultMeshRudpRejectionReason::MessageUnreadable(format!("{error:#}")), }), @@ -723,19 +742,24 @@ where && run.is_some_and(|run| run < MAX_CONSECUTIVE_REFUSED_FRAMES) { let refusal = rejection.refusal_message(self.receipts(key)); - let packets = match self.sessions.remove(&key) { - Some(mut entry) => { - let packets = self.refusal_packets(&mut entry.session, &refusal, false); - self.sessions.insert(key, entry); - packets - } - None => Vec::new(), - }; - for packet in &packets { - if let Some(error) = self.send_packet(key.remote_addr, packet)? { - unsendable = Some(error); - break; - } + if let Some(error) = self.send_on_live_session(key, &refusal, self.receipts(key))? { + unsendable = Some(error); + } + kept_rejection = Some(rejection); + continue; + } + // A refused write of a receipt-capable session is answered with its error.v1 and the session + // goes on: only a protocol violation (the cases above and below) ends a session. + if matches!(rejection.operation, CultMeshRudpApplicationOperation::DocumentPutRaw) + && self.receipts(key) + && !matches!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseSendFailed(_) + ) + { + let refusal = rejection.refusal_message(true); + if let Some(error) = self.send_on_live_session(key, &refusal, true)? { + unsendable = Some(error); } kept_rejection = Some(rejection); continue; @@ -928,6 +952,7 @@ where refused_deletes_in_a_row: 0, ending_deadline_monotonic_millis: None, receipts: false, + capability_settled: false, admitted_writes: HashMap::new(), }, ); @@ -964,29 +989,52 @@ where reason, })) }; - let content_hash = match write_content_hash( - "put", - &document.schema_id, - &document.record_key, - Some(&document.payload), - ) { - Ok(hash) => hash, - Err(error) => { - return reject(CultMeshRudpRejectionReason::ResponseEncodingFailed( - format!("{error:#}"), - )); - } + // The capability is fixed by the first hello or, absent one, the first write. + if let Some(entry) = self.sessions.get_mut(&key) { + entry.capability_settled = true; + } + let receipts = self.receipts(key); + let receipt = CultMeshRudpRawDocumentReceipt { + session: key, + message_id: message_id.clone(), + transport_sequence, + received_at_unix_millis: now_unix, + document, }; - let remembered = self - .sessions - .get(&key) - .filter(|entry| entry.receipts) - .map(|entry| (entry.admitted_writes.get(&message_id).copied(), entry.admitted_writes.len())); - if let Some((prior, remembered_count)) = remembered { + let admitted = CultNetMessage::WriteAdmitted { + message_id: message_id.clone(), + operation: "put".into(), + schema_id: receipt.document.schema_id.clone(), + record_key: receipt.document.record_key.clone(), + }; + let mut content_hash = [0_u8; 32]; + if receipts { + // Authorization comes first: a peer that may not write learns nothing of what this + // session remembers, not even that an id was used. + if let Err(error) = self.sink.authorize_raw_document(&receipt) { + return reject(CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}"))); + } + content_hash = match write_content_hash( + "put", + &receipt.document.schema_id, + &receipt.document.record_key, + Some(&receipt.document.payload), + ) { + Ok(hash) => hash, + Err(error) => { + return reject(CultMeshRudpRejectionReason::ResponseEncodingFailed( + format!("{error:#}"), + )); + } + }; + let (prior, remembered_count) = self + .sessions + .get(&key) + .map(|entry| (entry.admitted_writes.get(&message_id).copied(), entry.admitted_writes.len())) + .unwrap_or_default(); match prior { Some(prior) if prior == content_hash => { - self.send_write_admitted(key, now, &message_id, &document.schema_id, &document.record_key); - return Ok(None); + return self.receipt_sent(key, now, &admitted, reject); } Some(_) => return reject(CultMeshRudpRejectionReason::MessageIdReused), None if remembered_count >= MAX_REMEMBERED_WRITES_PER_SESSION => { @@ -995,14 +1043,13 @@ where None => {} } } - let (receipt_schema_id, receipt_record_key) = - (document.schema_id.clone(), document.record_key.clone()); + let document = &receipt.document; // Admit only what some snapshot request can return: the smallest // response that could carry this document is the record the snapshot // source would serve for it, alone, under the shortest message id // CultNet encodes, sized by the snapshot path's encoder and fragmented // as the snapshot path sends it. - let served = match self.snapshot_source.served_record(&document) { + let served = match self.snapshot_source.served_record(document) { Ok(served) => served, Err(error) => { return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed( @@ -1032,22 +1079,17 @@ where max_fragment_count, }); } - let receipt = CultMeshRudpRawDocumentReceipt { - session: key, - message_id: message_id.clone(), - transport_sequence, - received_at_unix_millis: now_unix, - document, - }; if let Err(error) = self.sink.accept_raw_document(receipt) { return reject(CultMeshRudpRejectionReason::SinkRefused(format!( "{error:#}" ))); } // Admitted: the sink holds it. Only now is a receipt owed, never before the commit point. - if let Some(entry) = self.sessions.get_mut(&key).filter(|entry| entry.receipts) { - entry.admitted_writes.insert(message_id.clone(), content_hash); - self.send_write_admitted(key, now, &message_id, &receipt_schema_id, &receipt_record_key); + if receipts { + if let Some(entry) = self.sessions.get_mut(&key) { + entry.admitted_writes.insert(message_id.clone(), content_hash); + } + return self.receipt_sent(key, now, &admitted, reject); } } CultNetMessage::Hello { @@ -1060,8 +1102,30 @@ where .flatten() .any(|candidate| candidate == version) }; - if let Some(entry) = self.sessions.get_mut(&key) { - entry.receipts = listed("cultnet.write_admitted.v0") && listed("cultnet.error.v1"); + let Some(entry) = self.sessions.get_mut(&key).filter(|entry| !entry.capability_settled) else { + return Ok(None); + }; + entry.capability_settled = true; + entry.receipts = listed(WRITE_ADMITTED_VERSION) && listed(ERROR_V1_VERSION); + if entry.receipts { + // The server's own hello tells the client this is a receipt server, not a legacy one + // that has not answered yet. Best effort: the client's window settles a missing hello. + let hello = CultNetMessage::Hello { + runtime_id: "cultmesh-rs-rudp-document-server".into(), + runtime_kind: "cultmesh-rs".into(), + agent_id: None, + role: None, + display_name: None, + supported_document_types: None, + supported_mutation_contracts: None, + supported_message_versions: Some(vec![ + WRITE_ADMITTED_VERSION.into(), + ERROR_V1_VERSION.into(), + ]), + transport_profiles: None, + supports_schema_catalog: None, + }; + self.send_reliable_schema(key, now, &hello); } } CultNetMessage::SnapshotRequest { @@ -1203,29 +1267,34 @@ where Ok(None) } - /// Sends the receipt for an admitted put. Best effort: the write is already committed, so a receipt that - /// cannot be queued or sent is dropped and the peer's own window settles the write as unconfirmed. It is - /// never answered with a refusal, which would make an admitted write look refused. - fn send_write_admitted( + /// Sends the receipt for an admitted put. A receipt that cannot be queued or sent would leave a committed + /// write unanswered, and a refusal would make it look refused; so the session ends instead, and the peer's + /// write settles as unconfirmed (`session_ended`). The write stays remembered. + fn receipt_sent( &mut self, key: CultMeshRudpSessionKey, now: u64, - message_id: &str, - schema_id: &str, - record_key: &str, - ) { - let receipt = CultNetMessage::WriteAdmitted { - message_id: message_id.into(), - operation: "put".into(), - schema_id: schema_id.into(), - record_key: record_key.into(), - }; - let Ok(payload) = encode_cultnet_message_to_vec(&receipt, CultNetWireContract::CultNetSchemaV0) else { - return; + receipt: &CultNetMessage, + reject: impl Fn(CultMeshRudpRejectionReason) -> Result>, + ) -> Result> { + if self.send_reliable_schema(key, now, receipt) { + return Ok(None); + } + self.end_session(key, SessionEnd::Refused)?; + reject(CultMeshRudpRejectionReason::ResponseSendFailed( + "the write receipt could not be sent".into(), + )) + } + + /// Queues `message` reliably and in order on the schema channel and sends it; false when it could not be + /// encoded, queued or sent. + fn send_reliable_schema(&mut self, key: CultMeshRudpSessionKey, now: u64, message: &CultNetMessage) -> bool { + let Ok(payload) = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0) else { + return false; }; let max_fragment_bytes = self.options.max_fragment_bytes; let Some(entry) = self.sessions.get_mut(&key) else { - return; + return false; }; let Ok(packets) = entry.session.send_many( "schema", @@ -1239,13 +1308,31 @@ where }, Some(max_fragment_bytes), ) else { - return; + return false; }; + packets + .iter() + .all(|packet| matches!(self.send_packet(key.remote_addr, packet), Ok(None))) + } + + /// Answers `refusal` on the session while it stays live; `Some` is a permanent send failure. + fn send_on_live_session( + &mut self, + key: CultMeshRudpSessionKey, + refusal: &CultNetMessage, + reliable: bool, + ) -> Result> { + let Some(mut entry) = self.sessions.remove(&key) else { + return Ok(None); + }; + let packets = self.refusal_packets(&mut entry.session, refusal, reliable); + self.sessions.insert(key, entry); for packet in &packets { - if !matches!(self.send_packet(key.remote_addr, packet), Ok(None)) { - break; + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + return Ok(Some(error)); } } + Ok(None) } /// Ends a session before its peer is done with it, and says goodbye. It is @@ -1288,11 +1375,11 @@ where Ok(()) } - /// The refusal a rejected peer is sent: unreliable and unordered, or (`reliable`) queued like any response. fn receipts(&self, key: CultMeshRudpSessionKey) -> bool { self.sessions.get(&key).is_some_and(|entry| entry.receipts) } + /// The refusal a rejected peer is sent: unreliable and unordered, or (`reliable`) queued like any response. fn refusal_packets( &self, session: &mut CultNetRudpSession, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 7eb6fcdc..c05eccf4 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -22,6 +22,8 @@ use std::time::Duration; struct SinkState { receipts: Vec, failures_remaining: usize, + /// Record keys the sink refuses to authorize, before and at the write. + denied_keys: Vec, } #[derive(Clone, Default)] @@ -34,11 +36,28 @@ impl Sink { ..Default::default() }))) } + + fn denying(keys: &[&str]) -> Self { + Self(Arc::new(Mutex::new(SinkState { + denied_keys: keys.iter().map(|key| key.to_string()).collect(), + ..Default::default() + }))) + } } impl CultMeshRudpRawDocumentSink for Sink { + fn authorize_raw_document(&mut self, receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + if self.0.lock().unwrap().denied_keys.contains(&receipt.document.record_key) { + anyhow::bail!("not authorized"); + } + Ok(()) + } + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { let mut state = self.0.lock().unwrap(); + if state.denied_keys.contains(&receipt.document.record_key) { + anyhow::bail!("not authorized"); + } if state.failures_remaining > 0 { state.failures_remaining -= 1; anyhow::bail!("injected sink failure CANARY-7f3a"); @@ -2112,6 +2131,36 @@ fn frames_after( Ok(frames) } +/// The hello a receipt server sends a receipt-capable client, ahead of any answer. +fn server_hello() -> CultNetMessage { + CultNetMessage::Hello { + runtime_id: "cultmesh-rs-rudp-document-server".into(), + runtime_kind: "cultmesh-rs".into(), + agent_id: None, + role: None, + display_name: None, + supported_document_types: None, + supported_mutation_contracts: None, + supported_message_versions: Some(vec![ + "cultnet.write_admitted.v0".into(), + "cultnet.error.v1".into(), + ]), + transport_profiles: None, + supports_schema_catalog: None, + } +} + +/// The `count` answers after the server's hello, which must come first. +fn answers_after( + server: &mut CultMeshRudpDocumentServer, + client: &mut CultNetRudpSocketTransportConnection, + count: usize, +) -> Result> { + let mut frames = frames_after(server, client, count + 1)?; + assert_eq!(frames.first(), Some(&server_hello()), "the server answers the hello first"); + Ok(frames.split_off(1)) +} + fn admitted(id: &str, key: &str) -> CultNetMessage { CultNetMessage::WriteAdmitted { message_id: id.into(), @@ -2129,7 +2178,7 @@ fn an_admitted_put_on_a_receipt_connection_is_answered_with_a_receipt_carrying_i connect(&mut server, &mut [&mut peer])?; send_reliable(&mut peer, &receipt_hello())?; send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; - assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); Ok(()) } @@ -2156,7 +2205,7 @@ fn a_resend_of_an_admitted_write_is_re_receipted_without_applying_it_twice() -> send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; assert_eq!( - frames_after(&mut server, &mut peer, 2)?, + answers_after(&mut server, &mut peer, 2)?, vec![admitted("m1", "note:1"), admitted("m1", "note:1")] ); assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); @@ -2172,7 +2221,7 @@ fn the_same_id_with_other_content_is_refused_message_invalid_never_re_receipted( send_reliable(&mut peer, &receipt_hello())?; send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; send_reliable(&mut peer, &put("m1", "note:1", vec![9, 9, 9]))?; - let frames = frames_after(&mut server, &mut peer, 2)?; + let frames = answers_after(&mut server, &mut peer, 2)?; assert_eq!(frames[0], admitted("m1", "note:1")); let CultNetMessage::ErrorV1 { code, details, message_id, .. } = &frames[1] else { panic!("the second answer must be a refusal, got {:?}", frames[1]); @@ -2192,7 +2241,7 @@ fn a_refused_put_on_a_receipt_connection_is_a_typed_error_naming_its_id() -> Res connect(&mut server, &mut [&mut peer])?; send_reliable(&mut peer, &receipt_hello())?; send_reliable(&mut peer, &put("m7", "note:7", vec![1]))?; - let frames = frames_after(&mut server, &mut peer, 1)?; + let frames = answers_after(&mut server, &mut peer, 1)?; let Some(CultNetMessage::ErrorV1 { code, message_id, .. }) = frames.first() else { panic!("expected error.v1, got {frames:?}"); }; @@ -2218,3 +2267,174 @@ fn a_hello_listing_only_one_of_the_two_frames_does_not_opt_in() -> Result<()> { } Ok(()) } + +fn refusal_of(message: &CultNetMessage) -> (Option, Option) { + let CultNetMessage::ErrorV1 { code, message_id, .. } = message else { + panic!("expected error.v1, got {message:?}"); + }; + (code.clone(), message_id.clone()) +} + +#[test] +fn a_refused_put_leaves_the_receipt_session_open_for_the_next_write() -> Result<()> { + let sink = Sink::fail_once(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 207)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m2", "note:2", vec![2]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::WriteNotAuthorized), Some("m1".into()))); + assert_eq!(answers[1], admitted("m2", "note:2")); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_reused_id_refuses_the_write_not_the_session() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 208)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![2]))?; + send_reliable(&mut peer, &put("m3", "note:3", vec![3]))?; + let answers = answers_after(&mut server, &mut peer, 3)?; + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("m1".into()))); + assert_eq!(answers[2], admitted("m3", "note:3")); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_full_dedupe_table_refuses_new_writes_and_keeps_the_session_and_its_receipts() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 209)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..1024 { + send_reliable(&mut peer, &put(&format!("w{index}"), &format!("note:{index}"), vec![1]))?; + if index % 16 == 15 { + for _ in 0..40 { + server.poll_once()?; + peer.poll_resends()?; + while peer.receive_once()?.is_some() {} + } + } + } + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + send_reliable(&mut peer, &put("w0", "note:0", vec![1]))?; + let answers = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (None, Some("over".into()))); + assert_eq!(answers[1], admitted("w0", "note:0"), "a remembered id is still answered idempotently"); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_readable_id_with_a_bad_field_is_refused_with_that_id() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 210)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + // A put with no document: it does not decode, but its id is there to read. + let frame = rmp_serde::to_vec_named(&serde_json::json!({ + "schemaVersion": "cultnet.document_put_raw.v0", + "messageId": "bad-1", + }))?; + peer.send_reliable("schema", frame)?; + peer.send_reliable("schema", garbage())?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("bad-1".into()))); + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), None), "no id can be read"); + Ok(()) +} + +#[test] +fn authorization_is_decided_before_a_reused_id_is_looked_at() -> Result<()> { + let sink = Sink::denying(&["secret"]); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 211)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "secret", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!( + refusal_of(&answers[1]), + (Some(CultNetErrorCode::WriteNotAuthorized), Some("m1".into())), + "a peer that may not write the key is not told the id was used" + ); + Ok(()) +} + +#[test] +fn a_second_hello_does_not_change_the_capability() -> Result<()> { + let mut downgrade = receipt_hello(); + if let CultNetMessage::Hello { supported_message_versions, .. } = &mut downgrade { + *supported_message_versions = None; + } + // Capable, then a hello listing nothing: still capable. + let sink = Sink::default(); + let mut server_a = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server_a.local_addr()?, 212)?; + connect(&mut server_a, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &downgrade)?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server_a, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + // Legacy by its first write, then a capable hello: still legacy. + let mut server_b = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut late = client(server_b.local_addr()?, 213)?; + connect(&mut server_b, &mut [&mut late])?; + send_reliable(&mut late, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut late, &receipt_hello())?; + send_reliable(&mut late, &put("m2", "note:2", vec![1]))?; + assert_eq!(frames_after(&mut server_b, &mut late, 1)?, vec![]); + Ok(()) +} + +#[test] +fn a_receipt_session_that_runs_out_of_payload_budget_is_ended_not_left_silent() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_admitted_payload_bytes_per_session: 400, + max_snapshot_response_bytes: 200, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 214)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..20 { + send_reliable(&mut peer, &put(&format!("w{index}"), &format!("note:{index}"), vec![1; 64]))?; + } + frames_after(&mut server, &mut peer, usize::MAX)?; + assert_eq!(server.session_count(), 0, "the writes the budget dropped were answered by ending the session"); + Ok(()) +} + +#[test] +fn a_receipt_that_cannot_be_queued_ends_the_session_instead_of_leaving_the_write_unanswered() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_pending_reliable_packets_per_session: 3, + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 215)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + // The peer never reads, so it acknowledges nothing and the server's reliable queue fills. + for index in 0..8 { + send_reliable(&mut peer, &put(&format!("w{index}"), &format!("note:{index}"), vec![1]))?; + for _ in 0..20 { + server.poll_once()?; + } + } + assert_eq!(server.session_count(), 0, "a receipt the queue could not take ended the session"); + Ok(()) +} diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index cc9a9c34..28ce7663 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -1979,3 +1979,17 @@ pub fn write_content_hash( rmpv::encode::write_value(&mut bytes, &value)?; Ok(Sha256::digest(&bytes).into()) } + +/// The `messageId` of a schema frame that does not decode as a message (write-receipt.md, rule 3: a refusal +/// carries the write's id whenever the frame is readable enough to find it). `None` when the bytes are not +/// a MessagePack map or hold no non-empty string `messageId`. +pub fn readable_message_id(bytes: &[u8]) -> Option { + let value = rmpv::decode::read_value(&mut &*bytes).ok()?; + let rmpv::Value::Map(entries) = value else { + return None; + }; + entries.into_iter().find_map(|(key, value)| match (key.as_str(), value) { + (Some("messageId"), rmpv::Value::String(id)) => id.into_str().filter(|id| !id.is_empty()), + _ => None, + }) +} diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 686c3bbe..259f2909 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -25,7 +25,7 @@ fn the_content_hash_matches_the_spec_vectors() { } #[test] -fn the_new_frames_encode_to_the_spec_bytes() { +fn the_new_frames_decode_back_equal() { let admitted = CultNetMessage::WriteAdmitted { message_id: "put-1".into(), operation: "put".into(), From 7a560b1984032c07268a469385b827674170d2fa Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 13:58:39 +0000 Subject: [PATCH 028/117] spec: dedupe scope is at least the session; a resend after a reconnect is a new write Per Heimdall's review. The dedupe key lives for the life of its session, bounded per session without forgetting an id the server promised to answer. A longer retention is optional and never to be relied on. After a reconnect callers resend only writes that are idempotent by record key and read back before resending any other. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 86f9f230..90acb8bb 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -28,11 +28,11 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir 2. **After the commit point.** `write_admitted` is sent only after the write is committed by the same rule that makes it visible to readers. Receipt of the bytes, parse, or authorisation alone is not admission. 3. **Id on every refusal.** Every refusal of a write, whatever its code (`write_not_authorized`, `message_invalid`, `unowned_schema`, `unsupported_schema_version`, shard authority errors, ...), is an `error.v1` whose `messageId` is the write's id. When the frame is readable enough to find the id, it is found; `message_invalid` for a missing or ill-typed field still carries the id. Nil only when no id can be read. 4. **Batches.** There is no batch frame; a batch is N writes, each with its own `messageId`, each answered under rules 1 to 3. A failed batch names its member by that id. A server may not collapse a batch into one answer. -5. **Identity and idempotent resend.** A `messageId` is unique per authenticated actor and session: the client mints it, never reuses it for different content in that session, and a server never accepts the id of one actor as the id of another. The server's dedupe key is `(actor, messageId)`, where *actor* is the transport-authenticated identity, never a name the hello or the frame claims. Alongside the key it keeps the **content hash** of the write and the answer it gave. The content hash is SHA-256 of the MessagePack array `[operation, schemaId, recordKey, payload]`, where `payload` is the raw document payload as a MessagePack `bin` for a put and nil for a delete (the vectors give worked hashes). It is retained at least `DEDUPE_RETENTION_MS` (10 minutes) and the life of the session. +5. **Identity and idempotent resend.** A `messageId` is unique per authenticated actor and session: the client mints it, never reuses it for different content in that session, and a server never accepts the id of one actor as the id of another. The server's dedupe key is `(actor, messageId)`, where *actor* is the transport-authenticated identity, never a name the hello or the frame claims. Alongside the key it keeps the **content hash** of the write and the answer it gave. The content hash is SHA-256 of the MessagePack array `[operation, schemaId, recordKey, payload]`, where `payload` is the raw document payload as a MessagePack `bin` for a put and nil for a delete (the vectors give worked hashes). **Scope: at least the session.** The key lives for the life of the session it was made in, and a server may bound how many writes it remembers per session (a session at the bound refuses new writes with an `error.v1` carrying the new write's id, and keeps answering the ids it remembers; it never forgets one it promised to answer). A server MAY keep the key longer for the same actor, up to `DEDUPE_RETENTION_MS` (10 minutes), but a client MUST NOT rely on it: **a reconnect is a new session, and a resend on the new session is a new write.** - Same key, same content hash, answer was `write_admitted`: answer `write_admitted` again with the same body. The write is not applied twice. - Same key, same content hash, answer was a refusal: the same refusal again. - **Same key, different content hash: refuse `message_invalid` (details `field: "messageId"`) and never re-send the old receipt.** Old receipts are for the old content; showing one for new content would report an unapplied write as admitted. The original write's state is unchanged. - - Past retention a resend is a new write. + - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. @@ -42,11 +42,11 @@ A write settles as exactly one of: - **admitted**: a `write_admitted` whose `messageId` is the write's. - **refused**: an `error.v1` whose `messageId` is the write's, carrying its `code` and `details` as the typed error. -- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window) and `session_ended` (the session ended first). Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content is safe against a receipt-capable server (rule 5); against a legacy server a resend may apply the write twice, so the caller reads the record back first. +- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window) and `session_ended` (the session ended first). Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. **Result window.** Two named constants (also in the vectors): `RECEIPT_RESULT_WINDOW_MS = 5000`, the default against a receipt-capable server (its hello lists both frames); `LEGACY_RESULT_WINDOW_MS = 500`, used only as the fallback against a server whose hello does not list them (or that sent none). The window runs from the send. A receipt-capable server that has not answered within its window is reported as unconfirmed like any other; clients log it as a server that missed its window. -**Late answers.** An answer for a write that already settled `unconfirmed` does not change what the caller was told, but the client records it on the write: a late `write_admitted` makes the write's recorded state `admitted_late`, a late refusal makes it `refused_late`. The write handle exposes that state, and a caller that got `unconfirmed` MUST consult it (and the record) before resending: it must not blind-resend a write that was admitted late. After a resend with the same id and content, rule 5 makes a duplicate harmless on a receipt-capable server. +**Late answers.** An answer for a write that already settled `unconfirmed` does not change what the caller was told, but the client records it on the write: a late `write_admitted` makes the write's recorded state `admitted_late`, a late refusal makes it `refused_late`. The write handle exposes that state, and a caller that got `unconfirmed` MUST consult it (and the record) before resending: it must not blind-resend a write that was admitted late. A resend on the same session with the same id and content is harmless on a receipt-capable server (rule 5); across a reconnect the caller follows the idempotence rule above. **Legacy answers.** An `error.v0`, or an `error.v1` with nil `messageId`, names no write. It is attributed to a write only when exactly one write on the connection is unanswered; it then refuses that write. With two or more unanswered writes it settles none of them and each ends unconfirmed at its window. A frame naming a `messageId` that is not in flight settles nothing. From b0ea18371fe4065e3f1f4ae38ecc8e66e596330c Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:04:37 +0000 Subject: [PATCH 029/117] C#: database server answers writes with write_admitted / error.v1 (write-receipt spec) Opt-in per connection by the first hello (or first write); the server answers a capable hello with its own. A put or delete commits, then gets write_admitted under its id; every refusal is error.v1 with the id (an unreadable frame carries the id when it still holds one). (actor, messageId) dedupe is bound to the content hash and asked only after the write authority grants; same id other content is message_invalid. The shard write forwarder says hello, waits for the primary's receipt, and the server relays only what the primary answered under the original id. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetDatabaseServer.cs | 166 +++++- .../CultNetShardWriteForwarding.cs | 43 +- src/GameCult.Networking/CultNetTransport.cs | 48 +- .../CultNetWriteReceipts.cs | 176 +++++- src/GameCult.Networking/Server.cs | 14 +- .../CultNetWriteReceiptServerTests.cs | 521 ++++++++++++++++++ .../NetworkingTests.cs | 55 +- 7 files changed, 985 insertions(+), 38 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 6433c6c3..f515b09d 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -22,6 +22,7 @@ namespace GameCult.Networking private readonly Func _snapshotHandlerV1; private readonly Func _putHandler; private readonly Func _deleteHandler; + private readonly Func _helloHandler; private readonly Func _shardCatalogHandler; private readonly Func _shardLogHandler; private readonly Func _subscribeHandler; @@ -45,6 +46,7 @@ namespace GameCult.Networking _snapshotHandlerV1 = HandleSnapshotRequestV1Async; _putHandler = HandlePutAsync; _deleteHandler = HandleDeleteAsync; + _helloHandler = HandleHelloAsync; _shardCatalogHandler = HandleShardCatalogRequestAsync; _shardLogHandler = HandleShardLogRequestAsync; _subscribeHandler = HandleSubscribeAsync; @@ -55,6 +57,7 @@ namespace GameCult.Networking _server.OnCultNet(_snapshotHandlerV1); _server.OnCultNet(_putHandler); _server.OnCultNet(_deleteHandler); + _server.OnCultNet(_helloHandler); _server.OnCultNet(_shardCatalogHandler); _server.OnCultNet(_shardLogHandler); _server.OnCultNet(_subscribeHandler); @@ -217,6 +220,7 @@ namespace GameCult.Networking _server.RemoveCultNetMessageListener(_snapshotHandlerV1); _server.RemoveCultNetMessageListener(_putHandler); _server.RemoveCultNetMessageListener(_deleteHandler); + _server.RemoveCultNetMessageListener(_helloHandler); _server.RemoveCultNetMessageListener(_shardCatalogHandler); _server.RemoveCultNetMessageListener(_shardLogHandler); _server.RemoveCultNetMessageListener(_subscribeHandler); @@ -323,74 +327,208 @@ namespace GameCult.Networking }; } + // A hello decides the connection's write framing if it is the first thing to (write-receipt.md, "Capability"). The + // server answers a receipt-capable hello with its own, so the client can tell a receipt server from a legacy one. + private Task HandleHelloAsync(CultNetHelloMessage hello, CultNetServerPeer peer) + { + if (peer.Transport != null && peer.Transport.SettleWriteFraming(hello)) + { + Answer(peer, new CultNetHelloMessage + { + RuntimeId = "gamecult-networking-database-server", + RuntimeKind = "dotnet", + SupportedMessageVersions = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1] + }); + } + + return Task.CompletedTask; + } + + // The write's receipt state, or null when its connection did not opt in (it is then answered exactly as before). + private static CultNetReceiptedWrite? BeginReceipted( + CultNetServerPeer peer, + string operation, + string? messageId, + string? schemaId, + string? recordKey, + byte[]? payload) + { + var transport = peer.Transport; + if (transport == null) return null; + transport.SettleWriteFraming(null); + var session = transport.WriteReceipts; + return session == null + ? null + : new CultNetReceiptedWrite(session, message => Answer(peer, message), operation, messageId, schemaId, recordKey, payload); + } + + // Every frame a write can be answered with. Mock point for tests: the answer leaves through the peer's transport. + internal static void Answer(CultNetServerPeer peer, ICultNetSchemaMessage message) + { + switch (message) + { + case CultNetWriteAdmittedMessage admitted: peer.SendCultNet(admitted); break; + case CultNetErrorV1Message error: peer.SendCultNet(error); break; + case CultNetErrorMessage error: peer.SendCultNet(error); break; + case CultNetHelloMessage hello: peer.SendCultNet(hello); break; + default: throw new ArgumentException($"{message.GetType().Name} is not a write answer.", nameof(message)); + } + } + + // A refusal: error.v1 carrying the write's id for a receipt-capable connection, the v0 error for any other. + private static void Refuse(CultNetServerPeer peer, CultNetReceiptedWrite? write, CultNetErrorMessage error) + { + if (write != null) write.Refused(error); + else peer.SendCultNet(error); + } + + // Rule 6: a forward is answered with what the primary answered, under the original id. A refusal the primary sent is + // relayed; silence or any other failure relays nothing, so the client settles it unconfirmed. + private void RelayForwardFailure(CultNetReceiptedWrite write, Exception failure) + { + if (failure is CultNetWriteRefusedException refused) + { + write.Refused(refused.Refusal); + return; + } + + _server.Logger.LogWarning($"CultNet forwarded write was not answered by its primary: {failure.Message}"); + write.Unanswered(); + } + private async Task HandlePutAsync(CultNetDocumentPutRawMessage message, CultNetServerPeer peer) { + var write = BeginReceipted(peer, "put", message.MessageId, message.Document?.SchemaId, message.Document?.RecordKey, message.Document?.Payload); + if (write != null && !write.HasId) + { + write.Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "a write needs a messageId")); + return; + } + + var authorize = Authorize(peer, CultNetRemoteWriteOperation.Put, message.ShardId, message.ShardEpoch); try { - await _database.ApplyPutAsync(message, Authorize(peer, CultNetRemoteWriteOperation.Put, message.ShardId, message.ShardEpoch)) - .ConfigureAwait(false); + await _database.ApplyPutAsync(message, write == null ? authorize : write.Guard(authorize)).ConfigureAwait(false); + write?.Admitted(); + } + catch (CultNetWriteResendException resend) when (write != null) + { + write.Resent(resend); } catch (CultNetRemoteWriteRefusedException ex) { _server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}"); - peer.SendCultNet(CultNetErrorMessage.ForRemoteWriteRefused(ex.Request, ex.Reason)); + Refuse(peer, write, CultNetErrorMessage.ForRemoteWriteRefused(ex.Request, ex.Reason)); } catch (CultNetShardAuthorityException ex) { - if (!await TryForwardPutAsync(ex, message).ConfigureAwait(false)) + bool forwarded; + try + { + forwarded = await TryForwardPutAsync(ex, message).ConfigureAwait(false); + } + catch (CultNetWriteUnconfirmedException) when (write == null) + { + // A peer that did not opt in is told nothing of a forward its primary left unanswered. + return; + } + catch (Exception failure) when (write != null) + { + RelayForwardFailure(write, failure); + return; + } + + if (forwarded) + { + write?.Admitted(); + } + else { _server.Logger.LogError($"CultNet raw put authority failed: {ex.Message}"); - peer.SendCultNet(CreateRoutingError(ex)); + Refuse(peer, write, CreateRoutingError(ex)); } } catch (CultNetUnownedSchemaException ex) { _server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}"); - peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + Refuse(peer, write, CultNetErrorMessage.ForUnownedSchema(ex)); } catch (Exception ex) when (CommittedWithLogFailure(ex)) { // The write committed and was published; only its shard log failed, so the peer is not told it was refused. _server.Logger.LogWarning($"CultNet raw put committed but its shard log failed: {ex.Message}"); + write?.Admitted(); } catch (Exception ex) { _server.Logger.LogError($"CultNet raw put failed: {ex.Message}"); - peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); + Refuse(peer, write, new CultNetErrorMessage { Error = ex.Message }); } } private async Task HandleDeleteAsync(CultNetDocumentDeleteMessage message, CultNetServerPeer peer) { + var write = BeginReceipted(peer, "delete", message.MessageId, message.SchemaId, message.RecordKey, null); + if (write != null && !write.HasId) + { + write.Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "a write needs a messageId")); + return; + } + + var authorize = Authorize(peer, CultNetRemoteWriteOperation.Delete, message.ShardId, message.ShardEpoch); try { - await _database.ApplyDeleteAsync(message, Authorize(peer, CultNetRemoteWriteOperation.Delete, message.ShardId, message.ShardEpoch)) - .ConfigureAwait(false); + await _database.ApplyDeleteAsync(message, write == null ? authorize : write.Guard(authorize)).ConfigureAwait(false); + write?.Admitted(); + } + catch (CultNetWriteResendException resend) when (write != null) + { + write.Resent(resend); } catch (CultNetRemoteWriteRefusedException ex) { _server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}"); - peer.SendCultNet(CultNetErrorMessage.ForRemoteWriteRefused(ex.Request, ex.Reason)); + Refuse(peer, write, CultNetErrorMessage.ForRemoteWriteRefused(ex.Request, ex.Reason)); } catch (CultNetShardAuthorityException ex) { - if (!await TryForwardDeleteAsync(ex, message).ConfigureAwait(false)) + bool forwarded; + try { - peer.SendCultNet(CreateRoutingError(ex)); + forwarded = await TryForwardDeleteAsync(ex, message).ConfigureAwait(false); + } + catch (CultNetWriteUnconfirmedException) when (write == null) + { + return; + } + catch (Exception failure) when (write != null) + { + RelayForwardFailure(write, failure); + return; + } + + if (forwarded) + { + write?.Admitted(); + } + else + { + Refuse(peer, write, CreateRoutingError(ex)); } } catch (CultNetUnownedSchemaException ex) { _server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}"); - peer.SendCultNet(CultNetErrorMessage.ForUnownedSchema(ex)); + Refuse(peer, write, CultNetErrorMessage.ForUnownedSchema(ex)); } catch (Exception ex) when (CommittedWithLogFailure(ex)) { _server.Logger.LogWarning($"CultNet raw delete committed but its shard log failed: {ex.Message}"); + write?.Admitted(); } catch (Exception ex) { - peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); + Refuse(peer, write, new CultNetErrorMessage { Error = ex.Message }); } } diff --git a/src/GameCult.Networking/CultNetShardWriteForwarding.cs b/src/GameCult.Networking/CultNetShardWriteForwarding.cs index 0fa6c4f7..cb390308 100644 --- a/src/GameCult.Networking/CultNetShardWriteForwarding.cs +++ b/src/GameCult.Networking/CultNetShardWriteForwarding.cs @@ -10,12 +10,15 @@ namespace GameCult.Networking public interface ICultNetShardWriteForwarder { /// - /// Forwards a raw document put to the shard owner. + /// Forwards a raw document put to the shard owner. Completes only when the owner answered that it admitted the write; + /// throws when it refused and + /// when it answered nothing. The server relays to the original peer exactly what the owner answered. /// Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message); /// - /// Forwards a raw document delete to the shard owner. + /// Forwards a raw document delete to the shard owner; completes, refuses and goes unconfirmed as + /// does. /// Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message); } @@ -70,10 +73,12 @@ namespace GameCult.Networking public Func? CreateClient { get; set; } /// - /// Gets or sets how long after a forwarded write the primary's refusal may still arrive; a refusal fails the - /// forward with . Defaults to . + /// Gets or sets how long the primary has to answer a forwarded write (from the send). Its receipt completes the forward, + /// its refusal fails it with , and silence fails it with + /// . Defaults to ; a + /// primary that predates write receipts never answers, so forwarding to one is unconfirmed after this window. /// - public TimeSpan ResultWindow { get; set; } = CultNetWriteCompletion.DefaultResultWindow; + public TimeSpan ResultWindow { get; set; } = CultNetWriteCompletion.ReceiptResultWindow; } /// @@ -92,8 +97,8 @@ namespace GameCult.Networking } /// - /// Forwards a put and completes on the primary's result frame: a refusal throws - /// , the same failure path as an unreachable primary. + /// Forwards a put under its original message id and completes on the primary's result frame: a receipt returns, a refusal + /// throws and silence throws . /// public async Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message) { @@ -101,12 +106,12 @@ namespace GameCult.Networking var endpoint = ResolvePrimaryEndpoint(shard); message.ShardId ??= shard.ShardId; message.ShardEpoch ??= shard.Epoch; - await SendAsync(endpoint, message).ConfigureAwait(false); + await SendAsync(endpoint, message, message.MessageId).ConfigureAwait(false); } /// - /// Forwards a delete and completes on the primary's result frame: a refusal throws - /// . + /// Forwards a delete under its original message id; completes, refuses and goes unconfirmed as + /// does. /// public async Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message) { @@ -114,7 +119,7 @@ namespace GameCult.Networking var endpoint = ResolvePrimaryEndpoint(shard); message.ShardId ??= shard.ShardId; message.ShardEpoch ??= shard.Epoch; - await SendAsync(endpoint, message).ConfigureAwait(false); + await SendAsync(endpoint, message, message.MessageId).ConfigureAwait(false); } /// @@ -138,19 +143,29 @@ namespace GameCult.Networking return (uri.Host, uri.IsDefaultPort ? 3075 : uri.Port); } - private async Task SendAsync(string endpoint, T message) where T : ICultNetSchemaMessage + private async Task SendAsync(string endpoint, T message, string messageId) where T : ICultNetSchemaMessage { var (host, port) = ParseEndpoint(endpoint); using var client = CreateClient(endpoint); client.Connect(host, port); await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false); - await CultNetWriteCompletion.SendAndAwaitResultAsync( - refuse => + // The hello comes first on the ordered schema channel: it opts this connection in to the primary's receipts. + client.SendCultNet(new CultNetHelloMessage + { + RuntimeId = "gamecult-networking-shard-forwarder", + RuntimeKind = "dotnet", + SupportedMessageVersions = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1] + }); + await CultNetWriteCompletion.SendAndAwaitReceiptAsync( + (admitted, refuse, refuseV1) => { + client.OnCultNet(admitted); client.OnCultNet(refuse); + client.OnCultNet(refuseV1); return null; }, () => client.SendCultNet(message), + messageId, _options.ResultWindow).ConfigureAwait(false); } diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index da465326..06e2d93b 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3533,7 +3533,7 @@ namespace GameCult.Networking /// public sealed class LiteNetLibTransportConnection { - private readonly NetPeer? _peer; + private readonly Action? _deliver; private readonly CultNetTransportStats _stats = new CultNetTransportStats(); /// @@ -3541,10 +3541,50 @@ namespace GameCult.Networking /// public LiteNetLibTransportConnection(NetPeer peer, CultNetTransportProfile? profile = null) { - _peer = peer ?? throw new ArgumentNullException(nameof(peer)); + if (peer == null) throw new ArgumentNullException(nameof(peer)); + _deliver = payload => peer.Send(payload, DeliveryMethod.ReliableOrdered); Profile = profile ?? CultNetTransportProfiles.CreateLiteNetLib("csharp-litenetlib-peer"); } + // The mock point for what leaves the connection: the peer's reliable ordered send. + internal LiteNetLibTransportConnection(Action deliver) + { + _deliver = deliver ?? throw new ArgumentNullException(nameof(deliver)); + Profile = CultNetTransportProfiles.CreateLiteNetLib("csharp-litenetlib-peer"); + } + + /// Answers a schema frame that did not decode with a message_invalid error.v1 naming its id when it holds one. + internal void AnswerUnreadableSchemaFrame(byte[] payload) => + SendSchema(CultNetErrorV1Message.From( + CultNetErrorMessage.ForMessageInvalid("message", "the message could not be read"), + CultNetReadableMessageId.From(payload))); + + private readonly object _writeFramingGate = new(); + private bool _writeFramingSettled; + + /// + /// The write-receipt state of a connection that opted in (write-receipt.md); null for every other connection, which is + /// answered exactly as before. + /// + internal CultNetWriteReceiptSession? WriteReceipts { get; private set; } + + /// + /// Decides this connection's write framing once: by its first hello, or by its first write if no hello came first. A + /// later hello changes nothing. The capability changes framing only; it never authorizes. Returns true when this call + /// opted the connection in. + /// + internal bool SettleWriteFraming(CultNetHelloMessage? hello) + { + lock (_writeFramingGate) + { + if (_writeFramingSettled) return false; + _writeFramingSettled = true; + if (hello == null || !CultNetWriteReceiptCapability.Declares(hello)) return false; + WriteReceipts = new CultNetWriteReceiptSession(); + return true; + } + } + /// /// Initializes a receive-only channel-aware adapter for LiteNetLib payload classification. /// @@ -3580,14 +3620,14 @@ namespace GameCult.Networking public void Send(string channelId, byte[] payload) { if (payload == null) throw new ArgumentNullException(nameof(payload)); - if (_peer == null) throw new InvalidOperationException("A LiteNetLib peer is required to send through this transport."); + if (_deliver == null) throw new InvalidOperationException("A LiteNetLib peer is required to send through this transport."); if (!string.Equals(channelId, "schema", StringComparison.Ordinal) && !string.Equals(channelId, "legacy", StringComparison.Ordinal)) { throw new InvalidOperationException($"litenetlib transport only supports schema and legacy channels, got \"{channelId}\"."); } - _peer.Send(payload, DeliveryMethod.ReliableOrdered); + _deliver(payload); _stats.BytesSent += payload.Length; _stats.FramesSent++; } diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index 2cc65db8..0d09a92f 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Security.Cryptography; +using GameCult.Caching; using MessagePack; namespace GameCult.Networking @@ -26,7 +27,8 @@ namespace GameCult.Networking if (payload == null) writer.WriteNil(); else writer.Write(payload); writer.Flush(); - return SHA256.HashData(buffer.WrittenSpan); + using var sha = SHA256.Create(); + return sha.ComputeHash(buffer.WrittenSpan.ToArray()); } } @@ -101,4 +103,176 @@ namespace GameCult.Networking } } } + + /// + /// The messageId of a schema frame that does not decode (write-receipt.md, rule 3: a refusal carries the write's id whenever + /// the frame is readable enough to find it). + /// + internal static class CultNetReadableMessageId + { + /// The frame's top-level string messageId, or null when the bytes are not a map holding one. + public static string? From(byte[] payload) + { + try + { + var reader = new MessagePackReader(payload); + var entries = reader.ReadMapHeader(); + for (var index = 0; index < entries; index++) + { + var key = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; + if (key == "messageId" && reader.NextMessagePackType == MessagePackType.String) + { + var id = reader.ReadString(); + return string.IsNullOrEmpty(id) ? null : id; + } + + if (key == null) reader.Skip(); + reader.Skip(); + } + } + catch (Exception) + { + // Not a readable map: no id. + } + + return null; + } + } + + /// A resend the receipt session answered without applying it again, or refused as a reused id. + internal sealed class CultNetWriteResendException : Exception + { + public CultNetWriteResendException(CultNetWriteReceiptSession.Begin outcome) + : base("write answered from the connection's remembered writes") + { + Outcome = outcome; + } + + public CultNetWriteReceiptSession.Begin Outcome { get; } + } + + /// + /// One write on a receipt-capable connection: its dedupe claim and its single answer (write-receipt.md, rules 1 to 5). + /// + internal sealed class CultNetReceiptedWrite + { + private readonly CultNetWriteReceiptSession _session; + private readonly Action _send; + private readonly string _operation; + private readonly string _messageId; + private readonly string _schemaId; + private readonly string _recordKey; + private readonly byte[]? _payload; + private bool _claimed; + private bool _answered; + + public CultNetReceiptedWrite( + CultNetWriteReceiptSession session, + Action send, + string operation, + string? messageId, + string? schemaId, + string? recordKey, + byte[]? payload) + { + _session = session; + _send = send; + _operation = operation; + _messageId = messageId ?? string.Empty; + _schemaId = schemaId ?? string.Empty; + _recordKey = recordKey ?? string.Empty; + _payload = payload; + } + + /// + /// Wraps the server's authorization so the connection's remembered writes are consulted only after the actor is + /// authorized: authorize first, then claim the id. A resend of an admitted write throws + /// instead of applying it again. + /// + public Action Guard(Action authorize) => (schemaId, key) => + { + authorize(schemaId, key); + if (_messageId.Length == 0) return; + var outcome = _session.TryBegin(_messageId, CultNetWriteContentHash.Compute(_operation, _schemaId, _recordKey, _payload)); + if (outcome == CultNetWriteReceiptSession.Begin.New) + { + _claimed = true; + return; + } + + throw new CultNetWriteResendException(outcome); + }; + + /// The write committed: remember it and send its one receipt. + public void Admitted() + { + if (_answered) return; + _answered = true; + if (_messageId.Length > 0) _session.Admitted(_messageId); + _send(new CultNetWriteAdmittedMessage + { + MessageId = _messageId, + Operation = _operation, + SchemaId = _schemaId, + RecordKey = _recordKey + }); + } + + /// The write was refused: forget its claim so a repeat is judged afresh, and send its one error.v1. + public void Refused(CultNetErrorMessage error) + { + if (_answered) return; + _answered = true; + Release(); + _send(CultNetErrorV1Message.From(error, _messageId)); + } + + /// No answer is owed or possible (a forward the primary never answered): forget the claim, send nothing. + public void Unanswered() + { + if (_answered) return; + _answered = true; + Release(); + } + + /// Answers a resend per rule 5. + public void Resent(CultNetWriteResendException resend) + { + switch (resend.Outcome) + { + case CultNetWriteReceiptSession.Begin.AdmittedBefore: + _answered = true; + _send(new CultNetWriteAdmittedMessage + { + MessageId = _messageId, + Operation = _operation, + SchemaId = _schemaId, + RecordKey = _recordKey + }); + break; + case CultNetWriteReceiptSession.Begin.InFlight: + // The first attempt is still being applied and will answer this id. + _answered = true; + break; + case CultNetWriteReceiptSession.Begin.DifferentContent: + Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "the message id was already used for different content")); + break; + default: + Refused(new CultNetErrorMessage { Error = "too many writes are remembered for this connection" }); + break; + } + } + + /// Whether the write has had its answer (or has none owed). + public bool Settled => _answered; + + /// A missing id cannot be answered under rule 1; refuse it with no id. + public bool HasId => _messageId.Length > 0; + + private void Release() + { + if (_claimed) _session.Abandoned(_messageId); + _claimed = false; + } + } } diff --git a/src/GameCult.Networking/Server.cs b/src/GameCult.Networking/Server.cs index 128eb47d..f158b1d2 100644 --- a/src/GameCult.Networking/Server.cs +++ b/src/GameCult.Networking/Server.cs @@ -406,7 +406,19 @@ namespace GameCult.Networking var frame = transport.Receive(bytes); if (string.Equals(frame.ChannelId, "schema", StringComparison.Ordinal)) { - var cultNetMessage = LiteNetLibTransportConnection.DecodeSchema(frame); + ICultNetSchemaMessage cultNetMessage; + try + { + cultNetMessage = LiteNetLibTransportConnection.DecodeSchema(frame); + } + catch (Exception ex) when (transport.WriteReceipts != null) + { + // A receipt-capable peer is told which frame was unreadable (by its id, if the frame still holds one). + Logger.LogWarning($"CultNet schema frame unreadable: {ex.Message}"); + transport.AnswerUnreadableSchemaFrame(frame.Payload); + return; + } + Logger.LogDebug($"Received CultNet schema message {cultNetMessage.SchemaVersion}"); await HandleCultNetSchemaMessageAsync(peer, user, cultNetMessage).ConfigureAwait(false); return; diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs new file mode 100644 index 00000000..9625c364 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -0,0 +1,521 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using GameCult.Caching; +using NUnit.Framework; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // The database server's half of contracts/cultnet/interop/write-receipt.md. A peer whose hello lists both new frames is + // answered with write_admitted or error.v1, each naming the write's id; any other peer is answered as before. The handlers + // are reached through their private delegates; a peer's transport is a capture of what leaves it. + public sealed class CultNetWriteReceiptServerTests + { + private const string ShardId = "receipt-shard"; + private static readonly CultRecordKey KeyA = new("tests:receipt:a"); + private static readonly CultRecordKey KeyB = new("tests:receipt:b"); + + private sealed class Authority : ICultNetRemoteWriteAuthority + { + public HashSet Denied { get; } = new(); + public CultNetRemoteWriteDecision Decide(CultNetRemoteWriteRequest request) => + Denied.Contains(request.RecordKey.Value) ? CultNetRemoteWriteDecision.Refuse("denied") : CultNetRemoteWriteDecision.Grant(); + } + + private sealed class Wire + { + public List Sent { get; } = new(); + public LiteNetLibTransportConnection Transport { get; } + public CultNetServerPeer Peer { get; } + + public Wire() + { + Transport = new LiteNetLibTransportConnection(bytes => + Sent.Add(LiteNetLibTransportConnection.DecodeSchema(LiteNetLibTransportConnection.Decode(bytes)))); + Peer = (CultNetServerPeer)RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); + typeof(CultNetServerPeer).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(Peer, Transport); + } + } + + private sealed class Rig + { + public CultCache Cache = null!; + public CultNetDatabase Database = null!; + public CultNetDatabaseServer DatabaseServer = null!; + public Authority Authority = null!; + public string SchemaId = string.Empty; + public Func PutHandler = null!; + public Func DeleteHandler = null!; + public Func HelloHandler = null!; + + public int LogLength => Database.GetMutationLog(ShardId).Count; + public string? TextOf(CultRecordKey key) => Cache.Get(key)?.Text; + + public CultNetDocumentPutRawMessage PutOf(string id, CultRecordKey key, string text) + { + var message = Database.Documents.CreateRawDocumentPutMessage(id, new CultRecordHandle(key), + new NetworkSchemaNote { Schema = "tests.networking_note.v1", Text = text }); + message.ShardId = ShardId; + message.ShardEpoch = 1; + return message; + } + + public CultNetDocumentDeleteMessage DeleteOf(string id, CultRecordKey key) => + new() { MessageId = id, SchemaId = SchemaId, RecordKey = key.Value, ShardId = ShardId, ShardEpoch = 1 }; + + public Task Put(Wire wire, string id, CultRecordKey key, string text) => PutHandler(PutOf(id, key, text), wire.Peer); + public Task Delete(Wire wire, string id, CultRecordKey key) => DeleteHandler(DeleteOf(id, key), wire.Peer); + public Task Hello(Wire wire, params string[] versions) => HelloHandler(HelloOf(versions), wire.Peer); + } + + private static CultNetHelloMessage HelloOf(params string[] versions) => + new() { RuntimeId = "test-client", RuntimeKind = "test", SupportedMessageVersions = versions }; + + private static readonly string[] BothFrames = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1]; + + private static Rig Build(Authority? authority = null, bool primary = true, ICultNetShardWriteForwarder? forwarder = null) + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = + [ + new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: primary, schemaIds: [schemaId], + primaryEndpoints: primary ? null : ["cultnet://runtime-owner:3075"]) + ] + }); + var host = new Server(cache, ServerSecurityOptions.Development()); + authority ??= new Authority(); + var databaseServer = new CultNetDatabaseServer(host, database, new CultNetDatabaseServerOptions + { + RemoteWriteAuthority = authority, + ForwardNonPrimaryWrites = forwarder != null, + WriteForwarder = forwarder + }); + const BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; + T Handler(string field) => (T)typeof(CultNetDatabaseServer).GetField(field, flags)!.GetValue(databaseServer)!; + return new Rig + { + Cache = cache, + Database = database, + DatabaseServer = databaseServer, + Authority = authority, + SchemaId = schemaId, + PutHandler = Handler>("_putHandler"), + DeleteHandler = Handler>("_deleteHandler"), + HelloHandler = Handler>("_helloHandler") + }; + } + + private static async Task<(Rig Rig, Wire Wire)> Capable() + { + var rig = Build(); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); // the server's own hello is checked on its own + return (rig, wire); + } + + private static CultNetWriteAdmittedMessage AssertAdmitted(ICultNetSchemaMessage message, string id, string operation, string recordKey) + { + var admitted = (CultNetWriteAdmittedMessage)message; + Assert.That(admitted.MessageId, Is.EqualTo(id)); + Assert.That(admitted.Operation, Is.EqualTo(operation)); + Assert.That(admitted.RecordKey, Is.EqualTo(recordKey)); + return admitted; + } + + private static CultNetErrorV1Message AssertRefused(ICultNetSchemaMessage message, string? id, string? code) + { + var error = (CultNetErrorV1Message)message; + Assert.That(error.MessageId, Is.EqualTo(id)); + Assert.That(error.Code, Is.EqualTo(code)); + return error; + } + + [Test] + public async Task AnAdmittedPutOnAReceiptConnectionIsAnsweredAfterItsCommit() + { + var (rig, wire) = await Capable(); + rig.Authority.Denied.Clear(); + + await rig.Put(wire, "m1", KeyA, "one"); + + Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); + var admitted = AssertAdmitted(wire.Sent.Single(), "m1", "put", KeyA.Value); + Assert.That(admitted.SchemaId, Is.EqualTo(rig.SchemaId)); + } + + [Test] + public async Task AnAdmittedDeleteIsAnsweredWithAReceiptNamingTheDelete() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "m1", KeyA, "one"); + wire.Sent.Clear(); + + await rig.Delete(wire, "d1", KeyA); + + Assert.That(rig.TextOf(KeyA), Is.Null); + AssertAdmitted(wire.Sent.Single(), "d1", "delete", KeyA.Value); + } + + [Test] + public async Task TheServerAnswersAReceiptHelloWithItsOwnListingBothFrames() + { + var rig = Build(); + var wire = new Wire(); + + await rig.Hello(wire, BothFrames); + + var hello = (CultNetHelloMessage)wire.Sent.Single(); + Assert.That(hello.SupportedMessageVersions, Is.EquivalentTo(BothFrames)); + } + + [TestCase("")] + [TestCase("cultnet.write_admitted.v0")] + [TestCase("cultnet.error.v1")] + public async Task APeerThatDoesNotListBothFramesIsAnsweredAsBeforeAndTheServerSaysNothingOfReceipts(string listed) + { + var rig = Build(); + var wire = new Wire(); + await rig.Hello(wire, listed.Length == 0 ? [] : [listed]); + await rig.Put(wire, "m1", KeyA, "one"); + + Assert.That(rig.TextOf(KeyA), Is.EqualTo("one"), "admitted all the same"); + Assert.That(wire.Sent, Is.Empty, "no receipt, no hello"); + + rig.Authority.Denied.Add(KeyB.Value); + await rig.Put(wire, "m2", KeyB, "two"); + Assert.That(wire.Sent.Single(), Is.TypeOf(), "a refusal is the v0 error, without an id"); + } + + [Test] + public async Task APeerThatNeverSaidHelloGetsNoReceipt() + { + var rig = Build(); + var wire = new Wire(); + + await rig.Put(wire, "m1", KeyA, "one"); + + Assert.That(wire.Sent, Is.Empty); + } + + [Test] + public async Task ASecondHelloDoesNotChangeTheCapability() + { + var (rig, wire) = await Capable(); + await rig.Hello(wire); + await rig.Put(wire, "m1", KeyA, "one"); + AssertAdmitted(wire.Sent.Single(), "m1", "put", KeyA.Value); + + var late = Build(); + var legacy = new Wire(); + await late.Put(legacy, "m1", KeyA, "one"); + await late.Hello(legacy, BothFrames); + await late.Put(legacy, "m2", KeyB, "two"); + Assert.That(legacy.Sent, Is.Empty, "a hello after the first write does not opt the connection in"); + } + + [Test] + public async Task ACapabilityNeverAuthorizes() + { + var (rig, wire) = await Capable(); + rig.Authority.Denied.Add(KeyA.Value); + + await rig.Put(wire, "m1", KeyA, "one"); + await rig.Delete(wire, "d1", KeyA); + + Assert.That(rig.TextOf(KeyA), Is.Null); + AssertRefused(wire.Sent[0], "m1", "write_not_authorized"); + AssertRefused(wire.Sent[1], "d1", "write_not_authorized"); + Assert.That(wire.Sent, Has.Count.EqualTo(2), "one answer per write"); + } + + [Test] + public async Task AResendOfAnAdmittedWriteIsReceiptedAgainWithoutBeingAppliedAgain() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "m1", KeyA, "one"); + var logAfterFirst = rig.LogLength; + + await rig.Put(wire, "m1", KeyA, "one"); + + Assert.That(wire.Sent, Has.Count.EqualTo(2)); + AssertAdmitted(wire.Sent[1], "m1", "put", KeyA.Value); + Assert.That(rig.LogLength, Is.EqualTo(logAfterFirst), "the resend was not committed again"); + } + + [Test] + public async Task TheSameIdWithOtherContentIsRefusedAndNeverReReceipted() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "m1", KeyA, "one"); + + await rig.Put(wire, "m1", KeyA, "other"); + await rig.Put(wire, "m1", KeyB, "one"); + await rig.Delete(wire, "m1", KeyA); + + Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); + Assert.That(rig.TextOf(KeyB), Is.Null); + Assert.That(wire.Sent, Has.Count.EqualTo(4)); + for (var index = 1; index < 4; index++) + { + var error = AssertRefused(wire.Sent[index], "m1", "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("messageId")); + } + } + + [Test] + public async Task AuthorizationIsDecidedBeforeAReusedIdIsLookedAt() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "m1", KeyA, "one"); + rig.Authority.Denied.Add(KeyB.Value); + + await rig.Put(wire, "m1", KeyB, "other"); + + AssertRefused(wire.Sent[1], "m1", "write_not_authorized"); + } + + [Test] + public async Task ARefusedWriteIsForgottenSoARepeatIsJudgedAfresh() + { + var (rig, wire) = await Capable(); + rig.Authority.Denied.Add(KeyA.Value); + await rig.Put(wire, "m1", KeyA, "one"); + rig.Authority.Denied.Clear(); + + await rig.Put(wire, "m1", KeyA, "one"); + + AssertRefused(wire.Sent[0], "m1", "write_not_authorized"); + AssertAdmitted(wire.Sent[1], "m1", "put", KeyA.Value); + Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); + } + + [Test] + public async Task AFullDedupeTableRefusesNewWritesAndStillAnswersRememberedOnes() + { + var (rig, wire) = await Capable(); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + { + await rig.Put(wire, $"w{index}", new CultRecordKey($"tests:receipt:{index}"), "x"); + } + + wire.Sent.Clear(); + await rig.Put(wire, "over", KeyA, "x"); + await rig.Put(wire, "w0", new CultRecordKey("tests:receipt:0"), "x"); + + AssertRefused(wire.Sent[0], "over", null); + AssertAdmitted(wire.Sent[1], "w0", "put", "tests:receipt:0"); + Assert.That(rig.TextOf(KeyA), Is.Null); + } + + [Test] + public async Task AWriteWithNoIdIsRefusedWithNoId() + { + var (rig, wire) = await Capable(); + + var message = rig.PutOf("placeholder", KeyA, "one"); + message.MessageId = string.Empty; + await rig.PutHandler(message, wire.Peer); + + var error = AssertRefused(wire.Sent.Single(), null, "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("messageId")); + Assert.That(rig.TextOf(KeyA), Is.Null); + } + + [Test] + public async Task AFrameThatDoesNotDecodeIsRefusedWithItsIdWhenItHoldsOne() + { + var (_, wire) = await Capable(); + var withId = MessagePack.MessagePackSerializer.Serialize(new Dictionary + { + ["schemaVersion"] = "cultnet.document_put_raw.v9", + ["messageId"] = "bad-1" + }); + + wire.Transport.AnswerUnreadableSchemaFrame(withId); + wire.Transport.AnswerUnreadableSchemaFrame(new byte[] { 0xC1 }); + + var error = AssertRefused(wire.Sent[0], "bad-1", "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("message")); + AssertRefused(wire.Sent[1], null, "message_invalid"); + } + + [Test] + public void ReadableMessageIdFindsOnlyANonEmptyStringIdInATopLevelMap() + { + string? IdOf(object value) => CultNetReadableMessageId.From(MessagePack.MessagePackSerializer.Serialize(value)); + + Assert.That(IdOf(new Dictionary { ["a"] = new[] { 1, 2 }, ["messageId"] = "x" }), Is.EqualTo("x")); + Assert.That(IdOf(new Dictionary { ["messageId"] = 5 }), Is.Null); + Assert.That(IdOf(new Dictionary { ["messageId"] = "" }), Is.Null); + Assert.That(IdOf(new Dictionary { ["nested"] = new Dictionary { ["messageId"] = "deep" } }), Is.Null); + Assert.That(IdOf("messageId"), Is.Null); + } + + [Test] + public void TheContentHashMatchesTheSpecVectors() + { + byte[] a = [0x81, 0xA2, 0x69, 0x64, 0xA1, 0x31]; + byte[] b = [0x81, 0xA2, 0x69, 0x64, 0xA1, 0x32]; + string Hex(byte[] hash) => Convert.ToHexString(hash).ToLowerInvariant(); + + Assert.That(Hex(CultNetWriteContentHash.Compute("put", "note.v0", "note:1", a)), Is.EqualTo("7051c10071123b6ade50fbd5075e3859cae4ea70ce358a24a0b0c6980090e95d")); + Assert.That(Hex(CultNetWriteContentHash.Compute("put", "note.v0", "note:1", b)), Is.EqualTo("58d3fc6b858d2605a68550c9d4811ccdca6b9db7a0c3bfebf304d279257f6f9f")); + Assert.That(Hex(CultNetWriteContentHash.Compute("put", "note.v0", "note:2", a)), Is.EqualTo("a248cd920e58d41bc6ccca9e108030b1d491239000047fef088122364a32b6f8")); + Assert.That(Hex(CultNetWriteContentHash.Compute("delete", "note.v0", "note:1", null)), Is.EqualTo("232b0f96651b28aa8ab7673a6106944a9f6a8d66852b6e21291a23ed084b8b37")); + } + + [Test] + public void TheNewFramesRoundTripOnTheWire() + { + var admitted = new CultNetWriteAdmittedMessage { MessageId = "m", Operation = "put", SchemaId = "s", RecordKey = "k" }; + var error = new CultNetErrorV1Message { Error = "e", Code = "message_invalid", MessageId = null }; + + var admittedBack = (CultNetWriteAdmittedMessage)CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize(admitted)); + var errorBack = (CultNetErrorV1Message)CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize(error)); + + Assert.That(admittedBack.MessageId, Is.EqualTo("m")); + Assert.That(admittedBack.RecordKey, Is.EqualTo("k")); + Assert.That(errorBack.MessageId, Is.Null); + Assert.That(errorBack.Code, Is.EqualTo("message_invalid")); + } + + private sealed class ScriptedForwarder : ICultNetShardWriteForwarder + { + public Func OnPut { get; set; } = _ => Task.CompletedTask; + public Func OnDelete { get; set; } = _ => Task.CompletedTask; + public int Forwarded { get; private set; } + + public Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message) + { + Forwarded++; + return OnPut(message); + } + + public Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message) + { + Forwarded++; + return OnDelete(message); + } + } + + [Test] + public async Task AForwardedWriteIsReceiptedUnderItsOriginalIdOnlyWhenThePrimaryAnswered() + { + var forwarder = new ScriptedForwarder(); + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + await rig.Delete(wire, "fwd-2", KeyA); + + AssertAdmitted(wire.Sent[0], "fwd-1", "put", KeyA.Value); + AssertAdmitted(wire.Sent[1], "fwd-2", "delete", KeyA.Value); + Assert.That(forwarder.Forwarded, Is.EqualTo(2)); + } + + [Test] + public async Task AForwardedWritesRefusalIsRelayedAsErrorV1UnderTheOriginalId() + { + var forwarder = new ScriptedForwarder + { + OnPut = _ => Task.FromException(new CultNetWriteRefusedException(new CultNetErrorMessage { Error = "primary says no", Code = "write_not_authorized" })), + OnDelete = _ => Task.FromException(new CultNetWriteRefusedException(CultNetErrorMessage.ForMessageInvalid("recordKey", "bad"))) + }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + await rig.Delete(wire, "fwd-2", KeyA); + + AssertRefused(wire.Sent[0], "fwd-1", "write_not_authorized"); + Assert.That(((CultNetErrorV1Message)wire.Sent[0]).Error, Is.EqualTo("primary says no")); + AssertRefused(wire.Sent[1], "fwd-2", "message_invalid"); + } + + [TestCase(true)] + [TestCase(false)] + public async Task AForwardThePrimaryNeverAnsweredRelaysNothingAndMayBeRetried(bool unconfirmed) + { + Exception failure = unconfirmed ? new CultNetWriteUnconfirmedException("fwd-1") : new TimeoutException("primary unreachable"); + var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(failure) }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + Assert.That(wire.Sent, Is.Empty, "no receipt, no error, no synthesized refusal"); + + forwarder.OnPut = _ => Task.CompletedTask; + await rig.Put(wire, "fwd-1", KeyA, "one"); + AssertAdmitted(wire.Sent.Single(), "fwd-1", "put", KeyA.Value); + Assert.That(forwarder.Forwarded, Is.EqualTo(2), "the unanswered attempt left no claim behind"); + } + + [Test] + public async Task APeerThatDidNotOptInIsToldNothingOfAnUnansweredForward() + { + var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(new CultNetWriteUnconfirmedException("fwd-1")) }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + + Assert.That(wire.Sent, Is.Empty); + } + + [Test] + public async Task AReceiptConnectionNotForwardableIsRefusedWithTheRoutingErrorAndItsId() + { + var rig = Build(primary: false); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + + var error = AssertRefused(wire.Sent.Single(), "fwd-1", null); + Assert.That(error.RoutingHint, Is.Not.Null); + } + + [Test] + public async Task AResendWhileTheFirstAttemptIsStillBeingAppliedIsNotAnsweredTwice() + { + var release = new TaskCompletionSource(); + var forwarder = new ScriptedForwarder { OnPut = _ => release.Task }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + var first = rig.Put(wire, "fwd-1", KeyA, "one"); + await rig.Put(wire, "fwd-1", KeyA, "one"); + Assert.That(wire.Sent, Is.Empty, "the resend waits for the first attempt's answer"); + release.SetResult(); + await first; + + AssertAdmitted(wire.Sent.Single(), "fwd-1", "put", KeyA.Value); + Assert.That(forwarder.Forwarded, Is.EqualTo(1), "and is not forwarded again"); + } + + [Test] + public void TheForwarderWaitsTheReceiptWindowByDefault() + { + Assert.That(new CultNetSchemaWriteForwarderOptions().ResultWindow, Is.EqualTo(CultNetWriteCompletion.ReceiptResultWindow)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 2a75b5c2..606d5625 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -5189,7 +5189,9 @@ namespace GameCult.Networking.Tests [Test] public async Task CultNetSchemaWriteForwarder_Uses_SchemaClientPort() { - var fake = new CapturingSchemaClient(_ => null); + var fake = new CapturingSchemaClient(message => message is CultNetDocumentDeleteMessage delete + ? new CultNetWriteAdmittedMessage { MessageId = delete.MessageId, Operation = "delete", SchemaId = delete.SchemaId, RecordKey = delete.RecordKey } + : null); var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions { CreateClient = () => fake @@ -5203,11 +5205,16 @@ namespace GameCult.Networking.Tests await forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { + MessageId = "fwd-del", SchemaId = "note.v0", RecordKey = "note:1" }); - var sent = (CultNetDocumentDeleteMessage)fake.SentMessages.Single(); + var hello = (CultNetHelloMessage)fake.SentMessages[0]; + Assert.That(hello.SupportedMessageVersions, Is.EquivalentTo(new[] { CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1 }), + "the hello opts the forwarding connection in before the write"); + var sent = (CultNetDocumentDeleteMessage)fake.SentMessages[1]; + Assert.That(sent.MessageId, Is.EqualTo("fwd-del"), "the write goes under its original id"); Assert.That(fake.ConnectedHost, Is.EqualTo("primary.example.test")); Assert.That(sent.ShardId, Is.EqualTo("notes")); Assert.That(sent.ShardEpoch, Is.EqualTo(3)); @@ -5241,7 +5248,7 @@ namespace GameCult.Networking.Tests } [Test] - public async Task CultNetSchemaWriteForwarder_DoesNotCompleteOnTheSendReturning() + public void CultNetSchemaWriteForwarder_SilenceIsUnconfirmedNotSuccess() { var fake = new CapturingSchemaClient(_ => null); var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions @@ -5254,11 +5261,51 @@ namespace GameCult.Networking.Tests primaryEndpoints: ["cultnet://primary.example.test:4075"]); var started = DateTime.UtcNow; - await forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { SchemaId = "note.v0", RecordKey = "note:1" }); + var unconfirmed = Assert.ThrowsAsync(() => + forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { MessageId = "fwd-del", SchemaId = "note.v0", RecordKey = "note:1" })); + Assert.That(unconfirmed!.MessageId, Is.EqualTo("fwd-del")); Assert.That(DateTime.UtcNow - started, Is.GreaterThanOrEqualTo(TimeSpan.FromMilliseconds(380))); } + [Test] + public void CultNetSchemaWriteForwarder_AnswersNamingAnotherWriteSettleNothing() + { + var fake = new CapturingSchemaClient(_ => null); + var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions + { + CreateClient = () => fake, + ResultWindow = TimeSpan.FromMilliseconds(300) + }); + var shard = new CultNetShardDescriptor( + "notes", "primary", 3, isPrimary: false, + primaryEndpoints: ["cultnet://primary.example.test:4075"]); + fake.OnSend = () => + { + fake.Deliver(new CultNetWriteAdmittedMessage { MessageId = "someone-else", Operation = "delete", SchemaId = "note.v0", RecordKey = "note:1" }); + fake.Deliver(new CultNetErrorV1Message { Error = "refused", Code = "write_not_authorized", MessageId = "someone-else" }); + }; + + Assert.ThrowsAsync(() => + forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { MessageId = "fwd-del", SchemaId = "note.v0", RecordKey = "note:1" })); + } + + [Test] + public void CultNetSchemaWriteForwarder_RejectsWithTheErrorV1NamingItsWrite() + { + var fake = new CapturingSchemaClient(_ => null); + var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions { CreateClient = () => fake }); + var shard = new CultNetShardDescriptor( + "notes", "primary", 3, isPrimary: false, + primaryEndpoints: ["cultnet://primary.example.test:4075"]); + fake.OnSend = () => fake.Deliver(new CultNetErrorV1Message { Error = "refused", Code = "write_not_authorized", MessageId = "fwd-del" }); + + var refused = Assert.ThrowsAsync(() => + forwarder.ForwardDeleteAsync(shard, new CultNetDocumentDeleteMessage { MessageId = "fwd-del", SchemaId = "note.v0", RecordKey = "note:1" })); + + Assert.That(refused!.Code, Is.EqualTo("write_not_authorized")); + } + [Test] public void CultMeshVerseCatalog_Finds_CompatibleTransferTargets() { From d4f440ac7c058db7f5b0a13d7578f8cbf9118521 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:22:50 +0000 Subject: [PATCH 030/117] cultmesh-rs: receipt server looks the id up first, remembers refusals, answers deletes and unqueueable refusals Follows Heimdall's second review of #60: replay by (session,id) before any authorization; refusals stored with their hash; delete shares the dedupe table and settles capability; a refusal the queue cannot take ends the session; a peer's refusal frames and a receipt session's refused deletes are not violations; the write limit has its own fixed text; the sink trait says Ok means committed and visible. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 362 ++++++++++++------ .../cultmesh-rs/tests/rudp_document_server.rs | 174 ++++++++- packages/cultnet-rs/tests/write_receipt.rs | 39 ++ 3 files changed, 430 insertions(+), 145 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 173ebf3b..1b90932b 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -10,6 +10,7 @@ use std::collections::{BTreeMap, HashMap}; use std::collections::BTreeSet; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; +use std::ops::ControlFlow; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const MAX_UDP_DATAGRAM_BYTES: usize = 65_535; @@ -66,14 +67,16 @@ pub struct CultMeshRudpSnapshotQuery { /// Caller-owned admission/persistence port for received raw documents. pub trait CultMeshRudpRawDocumentSink { - /// Whether this put may be written, decided without writing it. A receipt-capable session's resend or - /// reused id is checked against it before the session's remembered writes are consulted, so a peer that - /// may not write is refused as such and never shown what the session remembers (write-receipt.md). - /// The default admits everything and leaves the decision to `accept_raw_document`. + /// Whether this put may be written, decided without writing it. A receipt-capable session asks it only + /// the first time it sees a message id; a resend is answered from what the session remembers + /// (write-receipt.md, rule 5). The default admits everything and leaves the decision to + /// `accept_raw_document`. fn authorize_raw_document(&mut self, _receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { Ok(()) } + /// Commits the document. `Ok` means committed and visible to readers: a receipt-capable peer is told + /// `write_admitted` on the strength of it, so a sink that has only queued the write must not return `Ok`. fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()>; } @@ -269,6 +272,9 @@ pub enum CultMeshRudpRejectionReason { /// A receipt-capable session sent a message id it already used for a write with other content /// (write-receipt.md, rule 5). Nothing reached the sink; the first write's state is unchanged. MessageIdReused, + /// A receipt-capable session already remembers `MAX_REMEMBERED_WRITES_PER_SESSION` writes and takes no new + /// id. Nothing reached the sink. + WriteLimitReached, } impl std::fmt::Display for CultMeshRudpRejectionReason { @@ -278,6 +284,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { Self::DeleteNotSupported => f.write_str("the server does not take deletes"), Self::MessageUnreadable(error) => write!(f, "message could not be read: {error}"), Self::MessageIdReused => f.write_str("the message id was already used for different content"), + Self::WriteLimitReached => f.write_str("the session remembers as many writes as it will"), Self::DocumentUnservable { response_bytes, max_snapshot_response_bytes, @@ -318,7 +325,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { /// The texts a refused peer is sent, one per rejection variant. They are fixed: /// a sink's or source's error text can quote what a peer sent, so it stays with /// the caller that receives the rejection and never reaches the wire. -pub(crate) const REFUSAL_TEXTS: [&str; 12] = [ +pub(crate) const REFUSAL_TEXTS: [&str; 13] = [ "the catalog refused the document", "the document can never be served", "the snapshot source failed", @@ -331,6 +338,7 @@ pub(crate) const REFUSAL_TEXTS: [&str; 12] = [ "the server does not take deletes", "the message could not be read", "the message id was already used for different content", + "the session's remembered-write limit is full", ]; impl CultMeshRudpRejectionReason { @@ -349,6 +357,7 @@ impl CultMeshRudpRejectionReason { Self::DeleteNotSupported => 9, Self::MessageUnreadable(_) => 10, Self::MessageIdReused => 11, + Self::WriteLimitReached => 12, }] } } @@ -444,6 +453,15 @@ pub enum CultMeshRudpPollOutcome { ApplicationRejected(CultMeshRudpApplicationRejection), } +/// How a refusal sent on a live session fared. +enum Answered { + Sent, + /// The refusal can never be sent as built (a permanent send failure). + Unsendable(std::io::Error), + /// The session could not encode or queue the refusal: it was not sent at all. + Unqueueable, +} + /// Why a session ends before its peer is done with it. enum SessionEnd { /// The session refused a packet. @@ -454,6 +472,22 @@ enum SessionEnd { Unsendable(std::io::Error), } +fn write_rejection( + session: CultMeshRudpSessionKey, + operation: CultMeshRudpApplicationOperation, + message_id: &str, + record_key: &str, + reason: CultMeshRudpRejectionReason, +) -> CultMeshRudpApplicationRejection { + CultMeshRudpApplicationRejection { + session, + operation, + message_id: message_id.to_owned(), + record_key: Some(record_key.to_owned()), + reason, + } +} + struct SessionEntry { session: CultNetRudpSession, created_at_monotonic_millis: u64, @@ -470,13 +504,14 @@ struct SessionEntry { /// The capability is decided once, by the session's first hello or first write, whichever comes first; /// a later hello changes nothing. capability_settled: bool, - /// Admitted writes of a receipt-capable session: message id -> content hash, so a resend of the same - /// content is re-receipted without being applied again and the same id with other content is refused. - admitted_writes: HashMap, + /// Answered writes of a receipt-capable session: message id -> (content hash, the answer given), refusals + /// included, so a resend of the same content is answered again without being decided again and the same + /// id with other content is refused. + answered_writes: HashMap, } -/// The most admitted writes a receipt-capable session remembers; a session at the limit refuses new writes -/// (`PayloadBudgetFull`) rather than forget an id it promised to answer idempotently. +/// The most answered writes a receipt-capable session remembers; a session at the limit refuses new writes +/// (`WriteLimitReached`) rather than forget an id it promised to answer idempotently. const WRITE_ADMITTED_VERSION: &str = "cultnet.write_admitted.v0"; const ERROR_V1_VERSION: &str = "cultnet.error.v1"; @@ -698,9 +733,12 @@ where ); // Runs of refusals this frame extends or breaks. A session is kept through a short run and // ended by `MAX_CONSECUTIVE_REFUSED_FRAMES` in a row. + // Only an undecodable frame is a protocol violation of a receipt-capable session: its refused + // deletes are refused writes, answered one by one like its refused puts. + let receipt_session = self.receipts(key); let (is_unreadable, is_delete) = ( decoded.is_err(), - matches!(decoded, Ok(CultNetMessage::DocumentDelete { .. })), + !receipt_session && matches!(decoded, Ok(CultNetMessage::DocumentDelete { .. })), ); let run = self.sessions.get_mut(&key).map(|entry| { entry.undecodable_frames_in_a_row = if is_unreadable { @@ -742,24 +780,38 @@ where && run.is_some_and(|run| run < MAX_CONSECUTIVE_REFUSED_FRAMES) { let refusal = rejection.refusal_message(self.receipts(key)); - if let Some(error) = self.send_on_live_session(key, &refusal, self.receipts(key))? { - unsendable = Some(error); + match self.send_on_live_session(key, &refusal, self.receipts(key))? { + Answered::Sent => {} + Answered::Unsendable(error) => unsendable = Some(error), + Answered::Unqueueable => { + self.end_session(key, SessionEnd::Refused)?; + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } } kept_rejection = Some(rejection); continue; } // A refused write of a receipt-capable session is answered with its error.v1 and the session // goes on: only a protocol violation (the cases above and below) ends a session. - if matches!(rejection.operation, CultMeshRudpApplicationOperation::DocumentPutRaw) - && self.receipts(key) + if matches!( + rejection.operation, + CultMeshRudpApplicationOperation::DocumentPutRaw | CultMeshRudpApplicationOperation::DocumentDelete + ) && self.receipts(key) && !matches!( rejection.reason, CultMeshRudpRejectionReason::ResponseSendFailed(_) ) { let refusal = rejection.refusal_message(true); - if let Some(error) = self.send_on_live_session(key, &refusal, true)? { - unsendable = Some(error); + match self.send_on_live_session(key, &refusal, true)? { + Answered::Sent => {} + Answered::Unsendable(error) => unsendable = Some(error), + // An unanswerable write is answered by ending the session: its peer settles it + // as `session_ended`, never as silence. + Answered::Unqueueable => { + self.end_session(key, SessionEnd::Refused)?; + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } } kept_rejection = Some(rejection); continue; @@ -953,7 +1005,7 @@ where ending_deadline_monotonic_millis: None, receipts: false, capability_settled: false, - admitted_writes: HashMap::new(), + answered_writes: HashMap::new(), }, ); accept @@ -979,21 +1031,21 @@ where message_id, document, } => { - let record_key = document.record_key.clone(); - let reject = |reason| { - Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::DocumentPutRaw, - message_id: message_id.clone(), - record_key: Some(record_key.clone()), - reason, - })) - }; // The capability is fixed by the first hello or, absent one, the first write. if let Some(entry) = self.sessions.get_mut(&key) { entry.capability_settled = true; } let receipts = self.receipts(key); + let record_key = document.record_key.clone(); + let rejection = |reason| { + write_rejection( + key, + CultMeshRudpApplicationOperation::DocumentPutRaw, + &message_id, + &record_key, + reason, + ) + }; let receipt = CultMeshRudpRawDocumentReceipt { session: key, message_id: message_id.clone(), @@ -1001,95 +1053,46 @@ where received_at_unix_millis: now_unix, document, }; + if !receipts { + return Ok(self.admit_put(receipt, false).err().map(rejection)); + } let admitted = CultNetMessage::WriteAdmitted { message_id: message_id.clone(), operation: "put".into(), schema_id: receipt.document.schema_id.clone(), record_key: receipt.document.record_key.clone(), }; - let mut content_hash = [0_u8; 32]; - if receipts { - // Authorization comes first: a peer that may not write learns nothing of what this - // session remembers, not even that an id was used. - if let Err(error) = self.sink.authorize_raw_document(&receipt) { - return reject(CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}"))); - } - content_hash = match write_content_hash( - "put", - &receipt.document.schema_id, - &receipt.document.record_key, - Some(&receipt.document.payload), - ) { - Ok(hash) => hash, - Err(error) => { - return reject(CultMeshRudpRejectionReason::ResponseEncodingFailed( - format!("{error:#}"), - )); - } - }; - let (prior, remembered_count) = self - .sessions - .get(&key) - .map(|entry| (entry.admitted_writes.get(&message_id).copied(), entry.admitted_writes.len())) - .unwrap_or_default(); - match prior { - Some(prior) if prior == content_hash => { - return self.receipt_sent(key, now, &admitted, reject); - } - Some(_) => return reject(CultMeshRudpRejectionReason::MessageIdReused), - None if remembered_count >= MAX_REMEMBERED_WRITES_PER_SESSION => { - return reject(CultMeshRudpRejectionReason::PayloadBudgetFull); - } - None => {} - } - } - let document = &receipt.document; - // Admit only what some snapshot request can return: the smallest - // response that could carry this document is the record the snapshot - // source would serve for it, alone, under the shortest message id - // CultNet encodes, sized by the snapshot path's encoder and fragmented - // as the snapshot path sends it. - let served = match self.snapshot_source.served_record(document) { - Ok(served) => served, + let content_hash = match write_content_hash( + "put", + &receipt.document.schema_id, + &receipt.document.record_key, + Some(&receipt.document.payload), + ) { + Ok(hash) => hash, Err(error) => { - return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed( + return Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseEncodingFailed( format!("{error:#}"), - )); + )))); } }; - let alone = CultNetMessage::SnapshotResponseRaw { - message_id: SHORTEST_MESSAGE_ID.into(), - documents: vec![served], - }; - let response_bytes = match encode_snapshot_response(&alone) { - Ok(payload) => payload.len(), - Err(error) => { - return reject(CultMeshRudpRejectionReason::ResponseEncodingFailed(error)); - } - }; - let fragment_count = self.fragments_for(response_bytes); - let max_fragment_count = self.max_fragments_per_response(); - if response_bytes > self.options.max_snapshot_response_bytes - || fragment_count > max_fragment_count + // The id is looked up first: a resend is answered with what the session already said, and + // the sink is asked only about an id the session has not seen. + if let ControlFlow::Break(outcome) = + self.seen_write(key, now, &message_id, content_hash, &rejection)? { - return reject(CultMeshRudpRejectionReason::DocumentUnservable { - response_bytes, - max_snapshot_response_bytes: self.options.max_snapshot_response_bytes, - fragment_count, - max_fragment_count, - }); + return Ok(outcome); } - if let Err(error) = self.sink.accept_raw_document(receipt) { - return reject(CultMeshRudpRejectionReason::SinkRefused(format!( - "{error:#}" - ))); - } - // Admitted: the sink holds it. Only now is a receipt owed, never before the commit point. - if receipts { - if let Some(entry) = self.sessions.get_mut(&key) { - entry.admitted_writes.insert(message_id.clone(), content_hash); + match self.admit_put(receipt, true) { + Ok(()) => { + // Admitted: the sink holds it. Only now is a receipt owed, never before the commit point. + self.remember(key, &message_id, content_hash, admitted.clone()); + return self.answer_sent(key, now, &admitted, &rejection); + } + Err(reason) => { + let refused = rejection(reason); + self.remember(key, &message_id, content_hash, refused.refusal_message(true)); + return Ok(Some(refused)); } - return self.receipt_sent(key, now, &admitted, reject); } } CultNetMessage::Hello { @@ -1251,39 +1254,141 @@ where } CultNetMessage::DocumentDelete { message_id, + schema_id, record_key, - .. } => { - return Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::DocumentDelete, - message_id, - record_key: Some(record_key), - reason: CultMeshRudpRejectionReason::DeleteNotSupported, - })); + // A delete settles the connection's capability exactly as a put does. + if let Some(entry) = self.sessions.get_mut(&key) { + entry.capability_settled = true; + } + let rejection = |reason| { + write_rejection( + key, + CultMeshRudpApplicationOperation::DocumentDelete, + &message_id, + &record_key, + reason, + ) + }; + if !self.receipts(key) { + return Ok(Some(rejection(CultMeshRudpRejectionReason::DeleteNotSupported))); + } + let content_hash = match write_content_hash("delete", &schema_id, &record_key, None) { + Ok(hash) => hash, + Err(error) => { + return Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseEncodingFailed( + format!("{error:#}"), + )))); + } + }; + if let ControlFlow::Break(outcome) = + self.seen_write(key, now, &message_id, content_hash, &rejection)? + { + return Ok(outcome); + } + let refused = rejection(CultMeshRudpRejectionReason::DeleteNotSupported); + self.remember(key, &message_id, content_hash, refused.refusal_message(true)); + return Ok(Some(refused)); } _ => {} } Ok(None) } - /// Sends the receipt for an admitted put. A receipt that cannot be queued or sent would leave a committed - /// write unanswered, and a refusal would make it look refused; so the session ends instead, and the peer's - /// write settles as unconfirmed (`session_ended`). The write stays remembered. - fn receipt_sent( + /// What a receipt-capable session already knows of `message_id`. `Continue` is a first sighting that + /// the session has room to remember. A repeat with the same content is answered with the stored answer, + /// a repeat with other content (a put or a delete of any record) is `MessageIdReused`, and a session + /// that remembers its limit takes no new id. + fn seen_write( &mut self, key: CultMeshRudpSessionKey, now: u64, - receipt: &CultNetMessage, - reject: impl Fn(CultMeshRudpRejectionReason) -> Result>, + message_id: &str, + content_hash: [u8; 32], + rejection: &impl Fn(CultMeshRudpRejectionReason) -> CultMeshRudpApplicationRejection, + ) -> Result>> { + let (prior, remembered) = self + .sessions + .get(&key) + .map(|entry| (entry.answered_writes.get(message_id).cloned(), entry.answered_writes.len())) + .unwrap_or_default(); + Ok(ControlFlow::Break(match prior { + Some((hash, answer)) if hash == content_hash => self.answer_sent(key, now, &answer, rejection)?, + Some(_) => Some(rejection(CultMeshRudpRejectionReason::MessageIdReused)), + None if remembered >= MAX_REMEMBERED_WRITES_PER_SESSION => { + Some(rejection(CultMeshRudpRejectionReason::WriteLimitReached)) + } + None => return Ok(ControlFlow::Continue(())), + })) + } + + fn remember(&mut self, key: CultMeshRudpSessionKey, message_id: &str, content_hash: [u8; 32], answer: CultNetMessage) { + if let Some(entry) = self.sessions.get_mut(&key) { + entry.answered_writes.insert(message_id.to_owned(), (content_hash, answer)); + } + } + + /// Sends the answer owed a write. An answer that cannot be queued or sent would leave the write + /// unanswered, and a different answer would contradict what the sink did; so the session ends instead, + /// and the peer's write settles as unconfirmed (`session_ended`). The answer stays remembered. + fn answer_sent( + &mut self, + key: CultMeshRudpSessionKey, + now: u64, + answer: &CultNetMessage, + rejection: &impl Fn(CultMeshRudpRejectionReason) -> CultMeshRudpApplicationRejection, ) -> Result> { - if self.send_reliable_schema(key, now, receipt) { + if self.send_reliable_schema(key, now, answer) { return Ok(None); } self.end_session(key, SessionEnd::Refused)?; - reject(CultMeshRudpRejectionReason::ResponseSendFailed( - "the write receipt could not be sent".into(), - )) + Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseSendFailed( + "the write answer could not be sent".into(), + )))) + } + + /// Whether a put may be committed, committing it. `authorize` asks the sink first, as a receipt-capable + /// session does on an id's first sighting. + fn admit_put( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + authorize: bool, + ) -> std::result::Result<(), CultMeshRudpRejectionReason> { + if authorize { + if let Err(error) = self.sink.authorize_raw_document(&receipt) { + return Err(CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}"))); + } + } + let document = &receipt.document; + // Admit only what some snapshot request can return: the smallest + // response that could carry this document is the record the snapshot + // source would serve for it, alone, under the shortest message id + // CultNet encodes, sized by the snapshot path's encoder and fragmented + // as the snapshot path sends it. + let served = self + .snapshot_source + .served_record(document) + .map_err(|error| CultMeshRudpRejectionReason::SnapshotSourceFailed(format!("{error:#}")))?; + let alone = CultNetMessage::SnapshotResponseRaw { + message_id: SHORTEST_MESSAGE_ID.into(), + documents: vec![served], + }; + let response_bytes = encode_snapshot_response(&alone) + .map_err(CultMeshRudpRejectionReason::ResponseEncodingFailed)? + .len(); + let fragment_count = self.fragments_for(response_bytes); + let max_fragment_count = self.max_fragments_per_response(); + if response_bytes > self.options.max_snapshot_response_bytes || fragment_count > max_fragment_count { + return Err(CultMeshRudpRejectionReason::DocumentUnservable { + response_bytes, + max_snapshot_response_bytes: self.options.max_snapshot_response_bytes, + fragment_count, + max_fragment_count, + }); + } + self.sink + .accept_raw_document(receipt) + .map_err(|error| CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}"))) } /// Queues `message` reliably and in order on the schema channel and sends it; false when it could not be @@ -1315,24 +1420,27 @@ where .all(|packet| matches!(self.send_packet(key.remote_addr, packet), Ok(None))) } - /// Answers `refusal` on the session while it stays live; `Some` is a permanent send failure. + /// Answers `refusal` on the session while it stays live. fn send_on_live_session( &mut self, key: CultMeshRudpSessionKey, refusal: &CultNetMessage, reliable: bool, - ) -> Result> { + ) -> Result { let Some(mut entry) = self.sessions.remove(&key) else { - return Ok(None); + return Ok(Answered::Sent); }; let packets = self.refusal_packets(&mut entry.session, refusal, reliable); self.sessions.insert(key, entry); + if packets.is_empty() { + return Ok(Answered::Unqueueable); + } for packet in &packets { if let Some(error) = self.send_packet(key.remote_addr, packet)? { - return Ok(Some(error)); + return Ok(Answered::Unsendable(error)); } } - Ok(None) + Ok(Answered::Sent) } /// Ends a session before its peer is done with it, and says goodbye. It is diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index c05eccf4..645fb5df 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2329,6 +2329,8 @@ fn a_full_dedupe_table_refuses_new_writes_and_keeps_the_session_and_its_receipts send_reliable(&mut peer, &put("w0", "note:0", vec![1]))?; let answers = frames_after(&mut server, &mut peer, 2)?; assert_eq!(refusal_of(&answers[0]), (None, Some("over".into()))); + let CultNetMessage::ErrorV1 { error, .. } = &answers[0] else { unreachable!() }; + assert_eq!(error, "the session's remembered-write limit is full", "the cap has its own fixed text"); assert_eq!(answers[1], admitted("w0", "note:0"), "a remembered id is still answered idempotently"); assert_eq!(server.session_count(), 1); Ok(()) @@ -2354,24 +2356,6 @@ fn a_readable_id_with_a_bad_field_is_refused_with_that_id() -> Result<()> { Ok(()) } -#[test] -fn authorization_is_decided_before_a_reused_id_is_looked_at() -> Result<()> { - let sink = Sink::denying(&["secret"]); - let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; - let mut peer = client(server.local_addr()?, 211)?; - connect(&mut server, &mut [&mut peer])?; - send_reliable(&mut peer, &receipt_hello())?; - send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; - send_reliable(&mut peer, &put("m1", "secret", vec![1]))?; - let answers = answers_after(&mut server, &mut peer, 2)?; - assert_eq!( - refusal_of(&answers[1]), - (Some(CultNetErrorCode::WriteNotAuthorized), Some("m1".into())), - "a peer that may not write the key is not told the id was used" - ); - Ok(()) -} - #[test] fn a_second_hello_does_not_change_the_capability() -> Result<()> { let mut downgrade = receipt_hello(); @@ -2438,3 +2422,157 @@ fn a_receipt_that_cannot_be_queued_ends_the_session_instead_of_leaving_the_write assert_eq!(server.session_count(), 0, "a receipt the queue could not take ended the session"); Ok(()) } + +fn delete(id: &str, key: &str) -> CultNetMessage { + CultNetMessage::DocumentDelete { + message_id: id.into(), + schema_id: "test.raw.v1".into(), + record_key: key.into(), + } +} + +#[test] +fn a_resend_is_answered_from_memory_and_the_sink_is_not_asked_again() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 216)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + // The peer's authority over the key ends; the answer already given is still the answer. + sink.0.lock().unwrap().denied_keys.push("note:1".into()); + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + Ok(()) +} + +#[test] +fn a_used_id_with_other_content_is_message_invalid_even_for_a_key_the_peer_may_not_write() -> Result<()> { + let sink = Sink::denying(&["secret"]); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 211)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "secret", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("m1".into()))); + Ok(()) +} + +#[test] +fn a_refused_write_is_remembered_and_its_resend_replays_the_refusal() -> Result<()> { + let sink = Sink::fail_once(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 217)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::WriteNotAuthorized), Some("m1".into()))); + assert_eq!(answers[1], answers[0], "the same refusal again, not a fresh attempt that would now succeed"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + Ok(()) +} + +#[test] +fn a_delete_that_reuses_a_puts_id_is_message_invalid() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 218)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &delete("m1", "note:1"))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(answers[0], admitted("m1", "note:1")); + let CultNetMessage::ErrorV1 { code, details, message_id, .. } = &answers[1] else { + panic!("expected error.v1, got {:?}", answers[1]); + }; + assert_eq!(*code, Some(CultNetErrorCode::MessageInvalid)); + assert_eq!(message_id.as_deref(), Some("m1")); + assert_eq!(details.as_ref().and_then(|d| d.field.as_deref()), Some("messageId")); + Ok(()) +} + +#[test] +fn deletes_on_a_receipt_session_are_refused_one_by_one_and_never_end_it() -> Result<()> { + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 219)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..12 { + send_reliable(&mut peer, &delete(&format!("d{index}"), "note:1"))?; + } + send_reliable(&mut peer, &delete("d3", "note:1"))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 14)?; + assert_eq!(answers.len(), 14); + assert_eq!(refusal_of(&answers[12]), refusal_of(&answers[3]), "a resent delete replays its refusal"); + assert_eq!(answers[13], admitted("m1", "note:1")); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_delete_as_the_first_frame_settles_the_connection_as_legacy() -> Result<()> { + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 220)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &delete("d1", "note:1"))?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + let frames = frames_after(&mut server, &mut peer, 3)?; + assert_eq!(frames.len(), 1, "only the legacy delete refusal: {frames:?}"); + assert!(matches!(frames[0], CultNetMessage::Error { .. }), "{:?}", frames[0]); + Ok(()) +} + +#[test] +fn a_refusal_that_cannot_be_queued_ends_the_session_instead_of_vanishing() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_pending_reliable_packets_per_session: 3, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::denying(&["x"]), Source::default())?; + let mut peer = client(server.local_addr()?, 221)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..8 { + send_reliable(&mut peer, &put(&format!("w{index}"), "x", vec![1]))?; + for _ in 0..20 { + server.poll_once()?; + } + } + assert_eq!(server.session_count(), 0, "a refusal the queue could not take ended the session"); + Ok(()) +} + +#[test] +fn refusal_frames_from_a_peer_are_not_protocol_violations() -> Result<()> { + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 222)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for _ in 0..12 { + send_reliable( + &mut peer, + &CultNetMessage::ErrorV1 { + error: "message_invalid: nope".into(), + code: Some(CultNetErrorCode::MessageInvalid), + details: None, + message_id: Some("x".into()), + }, + )?; + send_reliable( + &mut peer, + &CultNetMessage::Error { error: "nope".into(), code: None, details: None }, + )?; + } + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(server.session_count(), 1); + Ok(()) +} diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 259f2909..20b3739a 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -50,3 +50,42 @@ fn the_new_frames_decode_back_equal() { error ); } + +fn unhex(text: &str) -> Vec { + (0..text.len()).step_by(2).map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap()).collect() +} + +/// The spec's wire vectors (write-receipt-vectors.json `wire`): the bytes decode to the vector's fields, not +/// merely to something that encodes back to itself. +#[test] +fn the_spec_wire_vectors_decode_to_their_expected_fields() { + let admitted = decode_cultnet_message_from_slice( + &unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"), + CultNetWireContract::CultNetSchemaV0, + ) + .unwrap(); + assert_eq!( + admitted, + CultNetMessage::WriteAdmitted { + message_id: "put-1".into(), + operation: "put".into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + } + ); + let refused = decode_cultnet_message_from_slice( + &unhex("86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93677726974655F6E6F745F617574686F72697A65643A2074686520636174616C6F6720726566757365642074686520646F63756D656E74AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A66E6F74653A32A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D32"), + CultNetWireContract::CultNetSchemaV0, + ) + .unwrap(); + let CultNetMessage::ErrorV1 { error, code, details, message_id } = refused else { + panic!("expected error.v1"); + }; + assert_eq!(error, "write_not_authorized: the catalog refused the document"); + assert_eq!(code, Some(CultNetErrorCode::WriteNotAuthorized)); + assert_eq!(message_id.as_deref(), Some("put-2")); + let details = details.unwrap(); + assert_eq!(details.field.as_deref(), Some("recordKey")); + assert_eq!(details.value.as_deref(), Some("note:2")); + assert_eq!((details.as_of, details.current), (None, None)); +} From 5f699c27fa5504ce403a42a66cb5811855227b6e Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:27:54 +0000 Subject: [PATCH 031/117] cultmesh-rs: a put reusing a delete's id is refused (mutant-driven test) Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 645fb5df..58e104a3 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2576,3 +2576,18 @@ fn refusal_frames_from_a_peer_are_not_protocol_violations() -> Result<()> { assert_eq!(server.session_count(), 1); Ok(()) } + +#[test] +fn a_put_that_reuses_a_deletes_id_is_message_invalid() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 223)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &delete("d1", "note:1"))?; + send_reliable(&mut peer, &put("d1", "note:1", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("d1".into()))); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "the reusing put was not applied"); + Ok(()) +} From eafcdcdf7c130b9539458c3506df7f47bcfdcd08 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:29:18 +0000 Subject: [PATCH 032/117] cultnet: add limit_reached to error.v1 code enum, with captured vector Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet/cultnet-distributed-database.md | 1 + .../interop/cultnet.error.v1.schema.json | 7 ++++--- .../cultnet/interop/write-receipt-vectors.json | 17 +++++++++++++++++ contracts/cultnet/interop/write-receipt.md | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index 686f4892..932ba861 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -115,6 +115,7 @@ copies), the C# `CultNetErrorMessage` factory, the Rust `CultNetErrorCode`, and `unowned_schema` (a write, or a remote put or delete, for a schema and key no shard owns), `write_not_authorized` (a peer's put or delete its server's remote write authority did not grant) and `unsupported_schema_version` (a message whose `schemaVersion` the peer does not speak; `details` names `schemaVersion` and the offered value) and +`limit_reached` (a receipt-capable server's per-session remembered-write table is full, so a new write is refused; `details` all null) and `message_invalid` (a request that lacks a required field or carries an ill-typed one: a document-put or delete, or a subscription without an id; `details.field` names the request field, or `message` when the whole frame cannot be read). A malformed put or delete is `message_invalid`, not a per-operation code: the peer knows which operation it sent. diff --git a/contracts/cultnet/interop/cultnet.error.v1.schema.json b/contracts/cultnet/interop/cultnet.error.v1.schema.json index 255cd6bb..cd9f47b5 100644 --- a/contracts/cultnet/interop/cultnet.error.v1.schema.json +++ b/contracts/cultnet/interop/cultnet.error.v1.schema.json @@ -45,7 +45,7 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each).", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each). limit_reached: a receipt-capable server refuses a NEW write because its per-session remembered-write table (1024 entries) is full; messageId is the write's id.", "type": [ "string", "null" @@ -60,7 +60,8 @@ "unowned_schema", "write_not_authorized", "unsupported_schema_version", - "message_invalid" + "message_invalid", + "limit_reached" ] }, "details": { @@ -78,7 +79,7 @@ ], "properties": { "field": { - "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\". message_invalid: the request field that is missing or ill-typed (\"message\" when the whole frame cannot be read).", + "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\". message_invalid: the request field that is missing or ill-typed (\"message\" when the whole frame cannot be read). limit_reached: no field; details stay all null.", "type": [ "string", "null" diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 93bfa4f3..f575398d 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -74,6 +74,23 @@ }, "messageId": null } + }, + { + "name": "error_v1_limit_reached", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93B6C696D69745F726561636865643A207468652073657373696F6E27732072656D656D62657265642D7772697465206C696D69742069732066756C6CAB726F7574696E6748696E74C0A4636F6465AD6C696D69745F72656163686564A764657461696C7384A56669656C64C0A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A87075742D31303235", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "limit_reached: the session's remembered-write limit is full", + "routingHint": null, + "code": "limit_reached", + "details": { + "field": null, + "value": null, + "asOf": null, + "current": null + }, + "messageId": "put-1025" + } } ], "schemaInvalid": [ diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 90acb8bb..4b97468f 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -32,6 +32,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - Same key, same content hash, answer was `write_admitted`: answer `write_admitted` again with the same body. The write is not applied twice. - Same key, same content hash, answer was a refusal: the same refusal again. - **Same key, different content hash: refuse `message_invalid` (details `field: "messageId"`) and never re-send the old receipt.** Old receipts are for the old content; showing one for new content would report an unapplied write as admitted. The original write's state is unchanged. + - **Remembered-write limit.** The per-session table of remembered writes holds at most 1024 entries. A server whose table is full refuses a NEW write (a key not already in the table) with `limit_reached`, `messageId` the write's id, `details` all null (no field is named); the write is not applied. A resend of a remembered key is still answered from the table. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. From ed5f9d30d136d4c50a2de0cdc664421096116a2c Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:32:39 +0000 Subject: [PATCH 033/117] C#: receipt server follows Heimdall's review of #61 Look the id up first and authorize only on first sighting; remember refusals within the cap; never re-forward an id whose primary stayed silent; end the connection when an answer cannot be queued; fixed refusal texts; limit_reached cap refusal; commit-point doc on ApplyPut/Delete. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetDatabase.cs | 3 + .../CultNetDatabaseServer.cs | 45 ++-- .../CultNetSchemaMessages.cs | 12 + src/GameCult.Networking/CultNetTransport.cs | 33 ++- .../CultNetWriteReceipts.cs | 175 +++++++------- .../CultNetWriteReceiptServerTests.cs | 218 ++++++++++++++++-- 6 files changed, 365 insertions(+), 121 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index d9e7ec09..af561ea5 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -1017,6 +1017,8 @@ namespace GameCult.Networking /// /// Applies a raw document mutation after checking shard authority. This is the host's own door (and replication's): /// it asks no remote write authority. A peer's put goes through , which does. + /// Returning normally means the put is committed and visible to readers: a receipt-capable peer is told + /// write_admitted on the strength of it, so a write that has only been queued must not return. /// /// No shard owns the schema and key; nothing was written. /// The change committed and was published, but the shard log could not record it. @@ -1047,6 +1049,7 @@ namespace GameCult.Networking /// /// Applies a raw document delete after checking shard authority. This is the host's own door: it asks no remote /// write authority. A peer's delete goes through , which does. + /// Returning normally means the delete is committed and visible to readers (see ). /// /// No shard owns the schema and key; nothing was written. /// The change committed and was published, but the shard log could not record it. diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index f515b09d..a6e55c3d 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -345,7 +345,7 @@ namespace GameCult.Networking } // The write's receipt state, or null when its connection did not opt in (it is then answered exactly as before). - private static CultNetReceiptedWrite? BeginReceipted( + private CultNetReceiptedWrite? BeginReceipted( CultNetServerPeer peer, string operation, string? messageId, @@ -362,8 +362,22 @@ namespace GameCult.Networking : new CultNetReceiptedWrite(session, message => Answer(peer, message), operation, messageId, schemaId, recordKey, payload); } - // Every frame a write can be answered with. Mock point for tests: the answer leaves through the peer's transport. - internal static void Answer(CultNetServerPeer peer, ICultNetSchemaMessage message) + // Every frame a write can be answered with. Mock point for tests: the answer leaves through the peer's transport. An + // answer that cannot be queued ends the connection: the peer settles its write as session_ended, never as silence. + private void Answer(CultNetServerPeer peer, ICultNetSchemaMessage message) + { + try + { + Send(peer, message); + } + catch (Exception ex) + { + _server.Logger.LogWarning($"CultNet write answer could not be sent; ending the connection: {ex.Message}"); + peer.Transport?.End(); + } + } + + private static void Send(CultNetServerPeer peer, ICultNetSchemaMessage message) { switch (message) { @@ -393,7 +407,10 @@ namespace GameCult.Networking } _server.Logger.LogWarning($"CultNet forwarded write was not answered by its primary: {failure.Message}"); - write.Unanswered(); + // Silence past the window means the primary may have applied it: never forward the id again on this session. + // Any other failure never reached a primary that could have. + if (failure is CultNetWriteUnconfirmedException) write.LeftSilent(); + else write.Unanswered(); } private async Task HandlePutAsync(CultNetDocumentPutRawMessage message, CultNetServerPeer peer) @@ -405,16 +422,14 @@ namespace GameCult.Networking return; } + if (write != null && !write.Begin()) return; + var authorize = Authorize(peer, CultNetRemoteWriteOperation.Put, message.ShardId, message.ShardEpoch); try { - await _database.ApplyPutAsync(message, write == null ? authorize : write.Guard(authorize)).ConfigureAwait(false); + await _database.ApplyPutAsync(message, authorize).ConfigureAwait(false); write?.Admitted(); } - catch (CultNetWriteResendException resend) when (write != null) - { - write.Resent(resend); - } catch (CultNetRemoteWriteRefusedException ex) { _server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}"); @@ -462,7 +477,7 @@ namespace GameCult.Networking catch (Exception ex) { _server.Logger.LogError($"CultNet raw put failed: {ex.Message}"); - Refuse(peer, write, new CultNetErrorMessage { Error = ex.Message }); + Refuse(peer, write, new CultNetErrorMessage { Error = CultNetWriteReceiptSession.WriteFailedText }); } } @@ -475,16 +490,14 @@ namespace GameCult.Networking return; } + if (write != null && !write.Begin()) return; + var authorize = Authorize(peer, CultNetRemoteWriteOperation.Delete, message.ShardId, message.ShardEpoch); try { - await _database.ApplyDeleteAsync(message, write == null ? authorize : write.Guard(authorize)).ConfigureAwait(false); + await _database.ApplyDeleteAsync(message, authorize).ConfigureAwait(false); write?.Admitted(); } - catch (CultNetWriteResendException resend) when (write != null) - { - write.Resent(resend); - } catch (CultNetRemoteWriteRefusedException ex) { _server.Logger.LogWarning($"CultNet raw delete refused: {ex.Message}"); @@ -528,7 +541,7 @@ namespace GameCult.Networking } catch (Exception ex) { - Refuse(peer, write, new CultNetErrorMessage { Error = ex.Message }); + Refuse(peer, write, new CultNetErrorMessage { Error = CultNetWriteReceiptSession.WriteFailedText }); } } diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 0a3f9adb..350ef43b 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -663,6 +663,18 @@ namespace GameCult.Networking }; } + /// + /// The refusal of a write a session has no room to remember (write-receipt.md, rule 5): code limit_reached, + /// details present and all nil. Exists only as error.v1; v0 has no such code. + /// + public static CultNetErrorV1Message LimitReached(string? messageId) => new CultNetErrorV1Message + { + Error = "limit_reached: the session's remembered-write limit is full", + Code = "limit_reached", + Details = new CultNetErrorDetails(), + MessageId = string.IsNullOrEmpty(messageId) ? null : messageId + }; + /// Gets the v0 form, dropping the id. public CultNetErrorMessage ToV0() => new CultNetErrorMessage { diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 06e2d93b..03cfef83 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3534,6 +3534,7 @@ namespace GameCult.Networking public sealed class LiteNetLibTransportConnection { private readonly Action? _deliver; + private readonly Action? _end; private readonly CultNetTransportStats _stats = new CultNetTransportStats(); /// @@ -3543,21 +3544,41 @@ namespace GameCult.Networking { if (peer == null) throw new ArgumentNullException(nameof(peer)); _deliver = payload => peer.Send(payload, DeliveryMethod.ReliableOrdered); + _end = peer.Disconnect; Profile = profile ?? CultNetTransportProfiles.CreateLiteNetLib("csharp-litenetlib-peer"); } // The mock point for what leaves the connection: the peer's reliable ordered send. - internal LiteNetLibTransportConnection(Action deliver) + internal LiteNetLibTransportConnection(Action deliver, Action? end = null) { _deliver = deliver ?? throw new ArgumentNullException(nameof(deliver)); + _end = end; Profile = CultNetTransportProfiles.CreateLiteNetLib("csharp-litenetlib-peer"); } - /// Answers a schema frame that did not decode with a message_invalid error.v1 naming its id when it holds one. - internal void AnswerUnreadableSchemaFrame(byte[] payload) => - SendSchema(CultNetErrorV1Message.From( - CultNetErrorMessage.ForMessageInvalid("message", "the message could not be read"), - CultNetReadableMessageId.From(payload))); + /// + /// Ends the connection. A write that can be neither answered nor refused is answered by this goodbye: the peer sees + /// its session end and settles the write as unconfirmed (session_ended), never as silence. + /// + internal void End() => _end?.Invoke(); + + /// + /// Answers a schema frame that did not decode with a message_invalid error.v1 naming its id when it holds one; + /// when that answer cannot be sent the connection ends instead. + /// + internal void AnswerUnreadableSchemaFrame(byte[] payload) + { + try + { + SendSchema(CultNetErrorV1Message.From( + CultNetErrorMessage.ForMessageInvalid("message", "the message could not be read"), + CultNetReadableMessageId.From(payload))); + } + catch (Exception) + { + End(); + } + } private readonly object _writeFramingGate = new(); private bool _writeFramingSettled; diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index 0d09a92f..87f2b286 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -48,14 +48,17 @@ namespace GameCult.Networking } /// - /// One receipt-capable peer's writes: message id to content hash, so a resend of the same content is re-receipted - /// without being committed again and the same id with other content is refused. + /// One receipt-capable peer's writes: message id to content hash and the answer given, refusals included, so a resend of the + /// same content is answered again without being decided again and the same id with other content is refused. /// internal sealed class CultNetWriteReceiptSession { - /// The most writes a peer is remembered for; a peer at the limit has new writes refused. + /// The most writes a peer is remembered for, refused ones included; a peer at the limit has new writes refused. internal const int MaxRememberedWrites = 1024; + /// What a write is refused with when it failed for a reason no typed refusal names. Never an exception's own text. + internal const string WriteFailedText = "the write could not be applied"; + private readonly object _gate = new(); private readonly Dictionary _writes = new(StringComparer.Ordinal); @@ -63,20 +66,31 @@ namespace GameCult.Networking { public Entry(byte[] hash) { Hash = hash; } public byte[] Hash { get; } - public bool Admitted { get; set; } + public bool Silent { get; set; } + public ICultNetSchemaMessage? Answer { get; set; } } - public enum Begin { New, AdmittedBefore, InFlight, DifferentContent, Full } + public enum Begin { New, Replay, InFlight, Silent, DifferentContent, Full } - /// Claims for a write with this content, or says what already holds it. - public Begin TryBegin(string messageId, byte[] hash) + /// + /// Looks up first. An id not seen is claimed for this content (); + /// otherwise the outcome says what already holds it, and is the answer to replay. + /// + public Begin TryBegin(string messageId, byte[] hash, out ICultNetSchemaMessage? answer) { lock (_gate) { + answer = null; if (_writes.TryGetValue(messageId, out var entry)) { if (!CryptographicOperations.FixedTimeEquals(entry.Hash, hash)) return Begin.DifferentContent; - return entry.Admitted ? Begin.AdmittedBefore : Begin.InFlight; + if (entry.Answer != null) + { + answer = entry.Answer; + return Begin.Replay; + } + + return entry.Silent ? Begin.Silent : Begin.InFlight; } if (_writes.Count >= MaxRememberedWrites) return Begin.Full; @@ -85,16 +99,28 @@ namespace GameCult.Networking } } - /// The write committed: remember it so a resend is re-receipted. - public void Admitted(string messageId) + /// The write was answered, admitted or refused: remember the answer so a resend repeats it. + public void Answered(string messageId, ICultNetSchemaMessage answer) { lock (_gate) { - if (_writes.TryGetValue(messageId, out var entry)) entry.Admitted = true; + if (_writes.TryGetValue(messageId, out var entry)) entry.Answer = answer; } } - /// The write was refused or unanswered: forget it so a corrected or repeated attempt is judged afresh. + /// + /// The primary left a forward unanswered: it may have applied it, so the id is never forwarded again on this session and + /// a resend is answered with nothing (the client settles it unconfirmed). + /// + public void LeftSilent(string messageId) + { + lock (_gate) + { + if (_writes.TryGetValue(messageId, out var entry)) entry.Silent = true; + } + } + + /// The write never reached anyone who could have applied it: forget it so a repeat is judged afresh. public void Abandoned(string messageId) { lock (_gate) @@ -139,18 +165,6 @@ namespace GameCult.Networking } } - /// A resend the receipt session answered without applying it again, or refused as a reused id. - internal sealed class CultNetWriteResendException : Exception - { - public CultNetWriteResendException(CultNetWriteReceiptSession.Begin outcome) - : base("write answered from the connection's remembered writes") - { - Outcome = outcome; - } - - public CultNetWriteReceiptSession.Begin Outcome { get; } - } - /// /// One write on a receipt-capable connection: its dedupe claim and its single answer (write-receipt.md, rules 1 to 5). /// @@ -185,94 +199,87 @@ namespace GameCult.Networking } /// - /// Wraps the server's authorization so the connection's remembered writes are consulted only after the actor is - /// authorized: authorize first, then claim the id. A resend of an admitted write throws - /// instead of applying it again. + /// Looks the write's id up before anything else is decided (rule 5). True when the id is new and the write should go on + /// to authorization and the commit; false when it was answered here (a replay, a reused id, a full table) or is owed no + /// further answer (still in flight, or left silent by its primary). /// - public Action Guard(Action authorize) => (schemaId, key) => + public bool Begin() { - authorize(schemaId, key); - if (_messageId.Length == 0) return; - var outcome = _session.TryBegin(_messageId, CultNetWriteContentHash.Compute(_operation, _schemaId, _recordKey, _payload)); - if (outcome == CultNetWriteReceiptSession.Begin.New) + var outcome = _session.TryBegin(_messageId, CultNetWriteContentHash.Compute(_operation, _schemaId, _recordKey, _payload), out var answer); + switch (outcome) { - _claimed = true; - return; + case CultNetWriteReceiptSession.Begin.New: + _claimed = true; + return true; + case CultNetWriteReceiptSession.Begin.Replay: + _answered = true; + _send(answer!); + return false; + case CultNetWriteReceiptSession.Begin.InFlight: + // The first attempt is still being applied and will answer this id. + _answered = true; + return false; + case CultNetWriteReceiptSession.Begin.Silent: + // The first attempt went to a primary that never answered; this id is never sent again, and stays unanswered. + _answered = true; + return false; + case CultNetWriteReceiptSession.Begin.DifferentContent: + Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "the message id was already used for different content")); + return false; + default: + _answered = true; + _send(CultNetErrorV1Message.LimitReached(_messageId)); + return false; } + } - throw new CultNetWriteResendException(outcome); - }; - - /// The write committed: remember it and send its one receipt. + /// The write committed: remember the receipt and send it. public void Admitted() { if (_answered) return; _answered = true; - if (_messageId.Length > 0) _session.Admitted(_messageId); - _send(new CultNetWriteAdmittedMessage + var receipt = new CultNetWriteAdmittedMessage { MessageId = _messageId, Operation = _operation, SchemaId = _schemaId, RecordKey = _recordKey - }); + }; + if (_claimed) _session.Answered(_messageId, receipt); + _send(receipt); } - /// The write was refused: forget its claim so a repeat is judged afresh, and send its one error.v1. + /// The write was refused: remember the refusal (a resend repeats it) and send its one error.v1. public void Refused(CultNetErrorMessage error) { if (_answered) return; _answered = true; - Release(); - _send(CultNetErrorV1Message.From(error, _messageId)); + var refusal = CultNetErrorV1Message.From(error, _messageId); + if (_claimed) _session.Answered(_messageId, refusal); + _send(refusal); } - /// No answer is owed or possible (a forward the primary never answered): forget the claim, send nothing. + /// + /// A forward its primary left unanswered: no answer is owed or possible, and the id is never forwarded again on this + /// session. Sends nothing. + /// + public void LeftSilent() + { + if (_answered) return; + _answered = true; + if (_claimed) _session.LeftSilent(_messageId); + } + + /// A forward that never reached a primary that could have applied it: forget the claim, send nothing. public void Unanswered() { if (_answered) return; _answered = true; - Release(); - } - - /// Answers a resend per rule 5. - public void Resent(CultNetWriteResendException resend) - { - switch (resend.Outcome) - { - case CultNetWriteReceiptSession.Begin.AdmittedBefore: - _answered = true; - _send(new CultNetWriteAdmittedMessage - { - MessageId = _messageId, - Operation = _operation, - SchemaId = _schemaId, - RecordKey = _recordKey - }); - break; - case CultNetWriteReceiptSession.Begin.InFlight: - // The first attempt is still being applied and will answer this id. - _answered = true; - break; - case CultNetWriteReceiptSession.Begin.DifferentContent: - Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "the message id was already used for different content")); - break; - default: - Refused(new CultNetErrorMessage { Error = "too many writes are remembered for this connection" }); - break; - } - } - - /// Whether the write has had its answer (or has none owed). - public bool Settled => _answered; - - /// A missing id cannot be answered under rule 1; refuse it with no id. - public bool HasId => _messageId.Length > 0; - - private void Release() - { if (_claimed) _session.Abandoned(_messageId); _claimed = false; } + + /// A missing id cannot be answered under rule 1; refuse it with no id. + public bool HasId => _messageId.Length > 0; } } diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 9625c364..b93b43b3 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -23,20 +23,32 @@ namespace GameCult.Networking.Tests private sealed class Authority : ICultNetRemoteWriteAuthority { public HashSet Denied { get; } = new(); - public CultNetRemoteWriteDecision Decide(CultNetRemoteWriteRequest request) => - Denied.Contains(request.RecordKey.Value) ? CultNetRemoteWriteDecision.Refuse("denied") : CultNetRemoteWriteDecision.Grant(); + public int Asked { get; private set; } + public CultNetRemoteWriteDecision Decide(CultNetRemoteWriteRequest request) + { + Asked++; + return Denied.Contains(request.RecordKey.Value) ? CultNetRemoteWriteDecision.Refuse("denied") : CultNetRemoteWriteDecision.Grant(); + } } private sealed class Wire { public List Sent { get; } = new(); + /// When set, the connection cannot queue what it is sent. + public bool Unsendable { get; set; } + public int Ended { get; private set; } public LiteNetLibTransportConnection Transport { get; } public CultNetServerPeer Peer { get; } public Wire() { - Transport = new LiteNetLibTransportConnection(bytes => - Sent.Add(LiteNetLibTransportConnection.DecodeSchema(LiteNetLibTransportConnection.Decode(bytes)))); + Transport = new LiteNetLibTransportConnection( + bytes => + { + if (Unsendable) throw new InvalidOperationException("the send queue is full"); + Sent.Add(LiteNetLibTransportConnection.DecodeSchema(LiteNetLibTransportConnection.Decode(bytes))); + }, + () => Ended++); Peer = (CultNetServerPeer)RuntimeHelpers.GetUninitializedObject(typeof(CultNetServerPeer)); typeof(CultNetServerPeer).GetField("k__BackingField", BindingFlags.NonPublic | BindingFlags.Instance)! .SetValue(Peer, Transport); @@ -273,7 +285,7 @@ namespace GameCult.Networking.Tests } [Test] - public async Task AuthorizationIsDecidedBeforeAReusedIdIsLookedAt() + public async Task AReusedIdIsJudgedBeforeAuthorityIsAskedAgain() { var (rig, wire) = await Capable(); await rig.Put(wire, "m1", KeyA, "one"); @@ -281,11 +293,25 @@ namespace GameCult.Networking.Tests await rig.Put(wire, "m1", KeyB, "other"); - AssertRefused(wire.Sent[1], "m1", "write_not_authorized"); + AssertRefused(wire.Sent[1], "m1", "message_invalid"); + Assert.That(rig.Authority.Asked, Is.EqualTo(1), "authority is asked on an id's first sighting only"); } [Test] - public async Task ARefusedWriteIsForgottenSoARepeatIsJudgedAfresh() + public async Task AResendIsAnsweredFromMemoryWithoutAskingAuthorityAgain() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "m1", KeyA, "one"); + rig.Authority.Denied.Add(KeyA.Value); + + await rig.Put(wire, "m1", KeyA, "one"); + + AssertAdmitted(wire.Sent[1], "m1", "put", KeyA.Value); + Assert.That(rig.Authority.Asked, Is.EqualTo(1)); + } + + [Test] + public async Task ARefusedWriteIsRememberedSoItsResendReplaysTheRefusal() { var (rig, wire) = await Capable(); rig.Authority.Denied.Add(KeyA.Value); @@ -294,9 +320,131 @@ namespace GameCult.Networking.Tests await rig.Put(wire, "m1", KeyA, "one"); - AssertRefused(wire.Sent[0], "m1", "write_not_authorized"); - AssertAdmitted(wire.Sent[1], "m1", "put", KeyA.Value); - Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); + var first = AssertRefused(wire.Sent[0], "m1", "write_not_authorized"); + var again = AssertRefused(wire.Sent[1], "m1", "write_not_authorized"); + Assert.That(again.Error, Is.EqualTo(first.Error)); + Assert.That(rig.TextOf(KeyA), Is.Null, "the resend was not decided afresh"); + Assert.That(rig.Authority.Asked, Is.EqualTo(1)); + } + + [Test] + public async Task ARefusedIdReusedForOtherContentIsMessageInvalid() + { + var (rig, wire) = await Capable(); + rig.Authority.Denied.Add(KeyA.Value); + await rig.Put(wire, "m1", KeyA, "one"); + + await rig.Put(wire, "m1", KeyB, "other"); + + AssertRefused(wire.Sent[1], "m1", "message_invalid"); + Assert.That(rig.TextOf(KeyB), Is.Null); + } + + [Test] + public async Task RefusedWritesCountAgainstTheRememberedWriteLimit() + { + var (rig, wire) = await Capable(); + rig.Authority.Denied.Add(KeyA.Value); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + { + await rig.Put(wire, $"r{index}", KeyA, $"v{index}"); + } + + wire.Sent.Clear(); + await rig.Put(wire, "over", KeyB, "x"); + await rig.Put(wire, "r0", KeyA, "v0"); + + AssertRefused(wire.Sent[0], "over", "limit_reached"); + AssertRefused(wire.Sent[1], "r0", "write_not_authorized"); + Assert.That(rig.TextOf(KeyB), Is.Null); + } + + [Test] + public async Task APutAndADeleteNeverShareAnId() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "p1", KeyA, "one"); + await rig.Delete(wire, "d1", KeyA); + wire.Sent.Clear(); + + await rig.Delete(wire, "p1", KeyA); + await rig.Put(wire, "d1", KeyA, "one"); + + AssertRefused(wire.Sent[0], "p1", "message_invalid"); + AssertRefused(wire.Sent[1], "d1", "message_invalid"); + Assert.That(rig.TextOf(KeyA), Is.Null, "neither reuse was applied"); + } + + [Test] + public async Task ADeleteAsTheFirstFrameSettlesTheConnectionAsLegacy() + { + var rig = Build(); + var wire = new Wire(); + await rig.Delete(wire, "d1", KeyA); + await rig.Hello(wire, BothFrames); + await rig.Put(wire, "m1", KeyA, "one"); + + Assert.That(wire.Sent, Is.Empty, "a hello after the first delete does not opt the connection in"); + } + + [Test] + public async Task AnAnswerThatCannotBeSentEndsTheConnectionInsteadOfLeavingTheWriteSilent() + { + var (rig, wire) = await Capable(); + wire.Unsendable = true; + await rig.Put(wire, "m1", KeyA, "one"); + Assert.That(wire.Ended, Is.EqualTo(1), "a receipt that cannot be queued"); + + rig.Authority.Denied.Add(KeyB.Value); + await rig.Put(wire, "m2", KeyB, "two"); + Assert.That(wire.Ended, Is.EqualTo(2), "a refusal that cannot be queued"); + + wire.Transport.AnswerUnreadableSchemaFrame(new byte[] { 0xC1 }); + Assert.That(wire.Ended, Is.EqualTo(3), "an unreadable-frame answer that cannot be queued"); + } + + [Test] + public async Task AnUnexpectedFailureIsRefusedWithFixedTextNotTheExceptionsOwn() + { + var (rig, wire) = await Capable(); + var message = rig.PutOf("m1", KeyA, "one"); + message.Document!.Payload = [0xC1]; + var legacy = new Wire(); + + await rig.PutHandler(message, wire.Peer); + await rig.PutHandler(message, legacy.Peer); + await rig.DeleteHandler(new CultNetDocumentDeleteMessage { MessageId = "d1", SchemaId = "no.such.schema", RecordKey = "k" }, legacy.Peer); + + Assert.That(((CultNetErrorV1Message)wire.Sent.Single()).Error, Is.EqualTo(CultNetWriteReceiptSession.WriteFailedText)); + Assert.That(((CultNetErrorMessage)legacy.Sent[0]).Error, Is.EqualTo(CultNetWriteReceiptSession.WriteFailedText)); + } + + [Test] + public void TheLimitReachedRefusalIsTheSpecVectorByteForByte() + { + const string vector = "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93B6C696D69745F726561636865643A207468652073657373696F6E27732072656D656D62657265642D7772697465206C696D69742069732066756C6CAB726F7574696E6748696E74C0A4636F6465AD6C696D69745F72656163686564A764657461696C7384A56669656C64C0A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A87075742D31303235"; + + var bytes = CultNetSchemaMessageSerialization.Serialize(CultNetErrorV1Message.LimitReached("put-1025")); + + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(vector)); + } + + [Test] + public void TheSpecWireVectorsDecodeToTheirExpectedFields() + { + var admitted = (CultNetWriteAdmittedMessage)CultNetSchemaMessageSerialization.Deserialize(Convert.FromHexString( + "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31")); + Assert.That((admitted.MessageId, admitted.Operation, admitted.SchemaId, admitted.RecordKey), + Is.EqualTo(("put-1", "put", "note.v0", "note:1"))); + + var refused = (CultNetErrorV1Message)CultNetSchemaMessageSerialization.Deserialize(Convert.FromHexString( + "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93677726974655F6E6F745F617574686F72697A65643A2074686520636174616C6F6720726566757365642074686520646F63756D656E74AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A66E6F74653A32A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D32")); + Assert.That(refused.Error, Is.EqualTo("write_not_authorized: the catalog refused the document")); + Assert.That(refused.Code, Is.EqualTo("write_not_authorized")); + Assert.That(refused.MessageId, Is.EqualTo("put-2")); + Assert.That((refused.Details!.Field, refused.Details.Value), Is.EqualTo(("recordKey", "note:2"))); + Assert.That(refused.Details.AsOf, Is.Null); + Assert.That(refused.Details.Current, Is.Null); } [Test] @@ -312,7 +460,8 @@ namespace GameCult.Networking.Tests await rig.Put(wire, "over", KeyA, "x"); await rig.Put(wire, "w0", new CultRecordKey("tests:receipt:0"), "x"); - AssertRefused(wire.Sent[0], "over", null); + var limit = AssertRefused(wire.Sent[0], "over", "limit_reached"); + Assert.That(limit.Error, Is.EqualTo("limit_reached: the session's remembered-write limit is full")); AssertAdmitted(wire.Sent[1], "w0", "put", "tests:receipt:0"); Assert.That(rig.TextOf(KeyA), Is.Null); } @@ -446,11 +595,50 @@ namespace GameCult.Networking.Tests AssertRefused(wire.Sent[1], "fwd-2", "message_invalid"); } - [TestCase(true)] - [TestCase(false)] - public async Task AForwardThePrimaryNeverAnsweredRelaysNothingAndMayBeRetried(bool unconfirmed) + [Test] + public async Task AForwardThePrimaryLeftSilentIsNeverForwardedAgainOnTheSession() { - Exception failure = unconfirmed ? new CultNetWriteUnconfirmedException("fwd-1") : new TimeoutException("primary unreachable"); + var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(new CultNetWriteUnconfirmedException("fwd-1")) }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + forwarder.OnPut = _ => Task.CompletedTask; + await rig.Put(wire, "fwd-1", KeyA, "one"); + + Assert.That(wire.Sent, Is.Empty, "no receipt, no error: the write stays unconfirmed"); + Assert.That(forwarder.Forwarded, Is.EqualTo(1), "the primary may have applied it, so it is not sent again"); + + await rig.Put(wire, "fwd-1", KeyB, "other"); + AssertRefused(wire.Sent.Single(), "fwd-1", "message_invalid"); + } + + [Test] + public async Task AForwardRefusedByThePrimaryReplaysItsRefusalOnResend() + { + var forwarder = new ScriptedForwarder + { + OnPut = _ => Task.FromException(new CultNetWriteRefusedException(new CultNetErrorMessage { Error = "primary says no", Code = "write_not_authorized" })) + }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + await rig.Put(wire, "fwd-1", KeyA, "one"); + + Assert.That(wire.Sent, Has.Count.EqualTo(2)); + Assert.That(((CultNetErrorV1Message)wire.Sent[1]).Error, Is.EqualTo("primary says no")); + Assert.That(forwarder.Forwarded, Is.EqualTo(1)); + } + + [Test] + public async Task AForwardThatNeverReachedThePrimaryMayBeRetried() + { + Exception failure = new TimeoutException("primary unreachable"); var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(failure) }; var rig = Build(primary: false, forwarder: forwarder); var wire = new Wire(); From 0182a6d40b48cd694fa503d53de8e499aab6b5f5 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:32:49 +0000 Subject: [PATCH 034/117] cultnet-rs, cultmesh-rs: remembered-write cap refusal is error.v1 limit_reached; pin authorize-once with a counting sink Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 5 ++ .../cultmesh-rs/tests/rudp_document_server.rs | 36 +++++++++++++- packages/cultnet-rs/src/contracts.rs | 11 +++++ packages/cultnet-rs/tests/write_receipt.rs | 47 ++++++++++++++++++- 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 1b90932b..c6825c8e 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -437,6 +437,11 @@ impl CultMeshRudpApplicationRejection { (CultMeshRudpRejectionReason::MessageIdReused, _) => { message_invalid_message("messageId", text) } + (CultMeshRudpRejectionReason::WriteLimitReached, _) => CultNetMessage::Error { + error: format!("limit_reached: {text}"), + code: Some(CultNetErrorCode::LimitReached), + details: Some(CultNetErrorDetails::default()), + }, _ => CultNetMessage::Error { error: text.into(), code: None, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 58e104a3..79e9e14c 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -24,6 +24,8 @@ struct SinkState { failures_remaining: usize, /// Record keys the sink refuses to authorize, before and at the write. denied_keys: Vec, + /// Every `authorize_raw_document` call, by message id. + authorized: Vec, } #[derive(Clone, Default)] @@ -47,6 +49,7 @@ impl Sink { impl CultMeshRudpRawDocumentSink for Sink { fn authorize_raw_document(&mut self, receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + self.0.lock().unwrap().authorized.push(receipt.message_id.clone()); if self.0.lock().unwrap().denied_keys.contains(&receipt.document.record_key) { anyhow::bail!("not authorized"); } @@ -2328,9 +2331,9 @@ fn a_full_dedupe_table_refuses_new_writes_and_keeps_the_session_and_its_receipts send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; send_reliable(&mut peer, &put("w0", "note:0", vec![1]))?; let answers = frames_after(&mut server, &mut peer, 2)?; - assert_eq!(refusal_of(&answers[0]), (None, Some("over".into()))); + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); let CultNetMessage::ErrorV1 { error, .. } = &answers[0] else { unreachable!() }; - assert_eq!(error, "the session's remembered-write limit is full", "the cap has its own fixed text"); + assert_eq!(error, "limit_reached: the session's remembered-write limit is full", "the cap has its own fixed text"); assert_eq!(answers[1], admitted("w0", "note:0"), "a remembered id is still answered idempotently"); assert_eq!(server.session_count(), 1); Ok(()) @@ -2477,6 +2480,35 @@ fn a_refused_write_is_remembered_and_its_resend_replays_the_refusal() -> Result< Ok(()) } +/// The sink is asked once, on an id's first sighting: not again for a resend (admitted or refused), and +/// not at all on a session that never declared the receipt capability. The sink here only counts, so a +/// flag that stopped or inverted the ask cannot hide behind the sink refusing at the write. +#[test] +fn authorize_is_called_once_per_first_sighting_and_only_for_receipt_sessions() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 219)?; + let mut legacy = client(server.local_addr()?, 220)?; + connect(&mut server, &mut [&mut peer, &mut legacy])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(sink.0.lock().unwrap().authorized, vec!["m1".to_string()], "first sighting asks once"); + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(sink.0.lock().unwrap().authorized, vec!["m1".to_string()], "the resend does not ask again"); + send_reliable(&mut legacy, &put("l1", "note:9", vec![1]))?; + for _ in 0..40 { + server.poll_once()?; + legacy.poll_resends()?; + while legacy.receive_once()?.is_some() {} + } + let state = sink.0.lock().unwrap(); + assert_eq!(state.receipts.len(), 2, "the legacy put was committed"); + assert_eq!(state.authorized, vec!["m1".to_string()], "a legacy session is never asked"); + Ok(()) +} + #[test] fn a_delete_that_reuses_a_puts_id_is_message_invalid() -> Result<()> { let sink = Sink::default(); diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 28ce7663..1aa39d11 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -249,6 +249,9 @@ pub enum CultNetErrorCode { WriteNotAuthorized, UnsupportedSchemaVersion, MessageInvalid, + /// `cultnet.error.v1` only: the session's remembered-write limit is full. `cultnet.error.v0` neither + /// encodes nor decodes it, so its enum stays the schema's. + LimitReached, } impl CultNetErrorCode { @@ -263,6 +266,7 @@ impl CultNetErrorCode { Self::WriteNotAuthorized => "write_not_authorized", Self::UnsupportedSchemaVersion => "unsupported_schema_version", Self::MessageInvalid => "message_invalid", + Self::LimitReached => "limit_reached", } } @@ -277,6 +281,7 @@ impl CultNetErrorCode { "write_not_authorized" => Ok(Self::WriteNotAuthorized), "unsupported_schema_version" => Ok(Self::UnsupportedSchemaVersion), "message_invalid" => Ok(Self::MessageInvalid), + "limit_reached" => Ok(Self::LimitReached), _ => Err(anyhow!("ErrorMessage.Code is not a recognized cultnet.error.v0 code")), } } @@ -1067,6 +1072,9 @@ fn parse_raw_cultnet_schema_message(input: &rmpv::Value) -> Result Result { + if let CultNetMessage::Error { code: Some(CultNetErrorCode::LimitReached), .. } = message { + return Err(anyhow!("limit_reached is a cultnet.error.v1 code; cultnet.error.v0 cannot carry it")); + } Ok(rmpv::Value::Map(match message { CultNetMessage::DocumentPutRaw { message_id, diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 20b3739a..4ad7a826 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -1,7 +1,7 @@ //! Wire parity with contracts/cultnet/interop/write-receipt-vectors.json: the content hash, and the two new frames. use cultnet_rs::{ - CultNetErrorCode, CultNetMessage, CultNetWireContract, decode_cultnet_message_from_slice, + CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetWireContract, decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, write_content_hash, }; @@ -89,3 +89,48 @@ fn the_spec_wire_vectors_decode_to_their_expected_fields() { assert_eq!(details.value.as_deref(), Some("note:2")); assert_eq!((details.as_of, details.current), (None, None)); } + +/// Spec vector `error_v1_limit_reached`: the bytes the server must send, and what they decode to. +#[test] +fn error_v1_limit_reached_is_the_spec_vector() { + let vector = "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93B6C696D69745F726561636865643A207468652073657373696F6E27732072656D656D62657265642D7772697465206C696D69742069732066756C6CAB726F7574696E6748696E74C0A4636F6465AD6C696D69745F72656163686564A764657461696C7384A56669656C64C0A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A87075742D31303235"; + let message = CultNetMessage::ErrorV1 { + error: "limit_reached: the session's remembered-write limit is full".into(), + code: Some(CultNetErrorCode::LimitReached), + details: Some(CultNetErrorDetails::default()), + message_id: Some("put-1025".into()), + }; + let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!(hex(&bytes), vector); + let CultNetMessage::ErrorV1 { error, code, details, message_id } = + decode_cultnet_message_from_slice(&unhex(vector), CultNetWireContract::CultNetSchemaV0).unwrap() + else { + panic!("expected error.v1"); + }; + assert_eq!(error, "limit_reached: the session's remembered-write limit is full"); + assert_eq!(code, Some(CultNetErrorCode::LimitReached)); + assert_eq!(message_id.as_deref(), Some("put-1025")); + assert_eq!(details, Some(CultNetErrorDetails::default())); +} + +/// `limit_reached` is in the error.v1 enum only: a v0 error neither encodes nor decodes it. +#[test] +fn error_v0_refuses_the_v1_only_limit_reached_code() { + let v0 = CultNetMessage::Error { + error: "x".into(), + code: Some(CultNetErrorCode::LimitReached), + details: None, + }; + assert!(encode_cultnet_message_to_vec(&v0, CultNetWireContract::CultNetSchemaV0).is_err()); + let v1 = CultNetMessage::ErrorV1 { + error: "x".into(), + code: Some(CultNetErrorCode::LimitReached), + details: None, + message_id: None, + }; + let v1_bytes = encode_cultnet_message_to_vec(&v1, CultNetWireContract::CultNetSchemaV0).unwrap(); + let at = v1_bytes.windows(16).position(|w| w == b"cultnet.error.v1").unwrap(); + let mut as_v0 = v1_bytes.clone(); + as_v0[at + 15] = b'0'; + assert!(decode_cultnet_message_from_slice(&as_v0, CultNetWireContract::CultNetSchemaV0).is_err()); +} From 74b8a4cbf8cfc556d7654434f0378db0b518aba3 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:53:05 +0000 Subject: [PATCH 035/117] C#: drop the redundant Silent state of a write claim (a silent claim is an unanswered one) Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetWriteReceipts.cs | 32 +++++-------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index 87f2b286..513efd99 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -66,11 +66,10 @@ namespace GameCult.Networking { public Entry(byte[] hash) { Hash = hash; } public byte[] Hash { get; } - public bool Silent { get; set; } public ICultNetSchemaMessage? Answer { get; set; } } - public enum Begin { New, Replay, InFlight, Silent, DifferentContent, Full } + public enum Begin { New, Replay, Unanswered, DifferentContent, Full } /// /// Looks up first. An id not seen is claimed for this content (); @@ -90,7 +89,8 @@ namespace GameCult.Networking return Begin.Replay; } - return entry.Silent ? Begin.Silent : Begin.InFlight; + // No answer yet: the first attempt is still being applied, or its primary left it unanswered for good. + return Begin.Unanswered; } if (_writes.Count >= MaxRememberedWrites) return Begin.Full; @@ -108,18 +108,6 @@ namespace GameCult.Networking } } - /// - /// The primary left a forward unanswered: it may have applied it, so the id is never forwarded again on this session and - /// a resend is answered with nothing (the client settles it unconfirmed). - /// - public void LeftSilent(string messageId) - { - lock (_gate) - { - if (_writes.TryGetValue(messageId, out var entry)) entry.Silent = true; - } - } - /// The write never reached anyone who could have applied it: forget it so a repeat is judged afresh. public void Abandoned(string messageId) { @@ -215,12 +203,9 @@ namespace GameCult.Networking _answered = true; _send(answer!); return false; - case CultNetWriteReceiptSession.Begin.InFlight: - // The first attempt is still being applied and will answer this id. - _answered = true; - return false; - case CultNetWriteReceiptSession.Begin.Silent: - // The first attempt went to a primary that never answered; this id is never sent again, and stays unanswered. + case CultNetWriteReceiptSession.Begin.Unanswered: + // The first attempt is still being applied and will answer this id, or its primary left it unanswered: + // then the claim stays, so the id is never sent again on this session, and the write stays unconfirmed. _answered = true; return false; case CultNetWriteReceiptSession.Begin.DifferentContent: @@ -260,14 +245,13 @@ namespace GameCult.Networking } /// - /// A forward its primary left unanswered: no answer is owed or possible, and the id is never forwarded again on this - /// session. Sends nothing. + /// A forward its primary left unanswered: no answer is owed or possible, and the claim stays so the id is never + /// forwarded again on this session. Sends nothing. /// public void LeftSilent() { if (_answered) return; _answered = true; - if (_claimed) _session.LeftSilent(_messageId); } /// A forward that never reached a primary that could have applied it: forget the claim, send nothing. From 2e5df24e96e54c4edd4e3455d700538e11c39ed5 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:58:17 +0000 Subject: [PATCH 036/117] C#: loopback test for the unreadable-frame answer; Server.Port seam; the answer now reaches the wire An undecodable frame is classed legacy by Decode, so the schema-side guard was unreachable and a receipt peer was never answered. The guard moves to the legacy decode. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/Server.cs | 44 ++-- .../CultNetWriteReceiptLoopbackTests.cs | 188 ++++++++++++++++++ 2 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs diff --git a/src/GameCult.Networking/Server.cs b/src/GameCult.Networking/Server.cs index f158b1d2..d9b763b9 100644 --- a/src/GameCult.Networking/Server.cs +++ b/src/GameCult.Networking/Server.cs @@ -23,7 +23,7 @@ namespace GameCult.Networking private const string EmailPattern = @"^([0-9a-zA-Z]([\+\-_\.][0-9a-zA-Z]+)*)+@(([0-9a-zA-Z][-\w]*[0-9a-zA-Z]*\.)+[a-zA-Z0-9]{2,17})$"; private const string UsernamePattern = @"^[A-Za-z0-9]+(?:[ _-][A-Za-z0-9]+)*$"; - private const int ServerPort = 3075; + private const int DefaultPort = 3075; private const float SessionTimeoutSeconds = 1800; // 30 minutes private const float SessionRefreshThresholdSeconds = 300; // 5 minutes private const int MaxConnectionAttemptsPerMinute = 30; @@ -63,6 +63,12 @@ namespace GameCult.Networking /// public bool LogSensitivePayloads { get; set; } + /// + /// Gets or sets the UDP port binds (default 3075; 0 asks the OS for a free one). After + /// it holds the port actually bound. + /// + public int Port { get; set; } = DefaultPort; + private float Time => (float)(_timer?.Elapsed.TotalSeconds ?? 0d); /// @@ -74,7 +80,7 @@ namespace GameCult.Networking { TransportId = "litenetlib-server", Host = "0.0.0.0", - Port = ServerPort + Port = Port }); /// @@ -357,7 +363,8 @@ namespace GameCult.Networking UnsyncedEvents = true, NatPunchEnabled = true }; - _netManager.Start(ServerPort); + _netManager.Start(Port); + Port = _netManager.LocalPort; listener.NetworkErrorEvent += (point, code) => Logger.LogInfo($"{point.Address}: Error {code}"); @@ -406,25 +413,26 @@ namespace GameCult.Networking var frame = transport.Receive(bytes); if (string.Equals(frame.ChannelId, "schema", StringComparison.Ordinal)) { - ICultNetSchemaMessage cultNetMessage; - try - { - cultNetMessage = LiteNetLibTransportConnection.DecodeSchema(frame); - } - catch (Exception ex) when (transport.WriteReceipts != null) - { - // A receipt-capable peer is told which frame was unreadable (by its id, if the frame still holds one). - Logger.LogWarning($"CultNet schema frame unreadable: {ex.Message}"); - transport.AnswerUnreadableSchemaFrame(frame.Payload); - return; - } - + var cultNetMessage = LiteNetLibTransportConnection.DecodeSchema(frame); Logger.LogDebug($"Received CultNet schema message {cultNetMessage.SchemaVersion}"); await HandleCultNetSchemaMessageAsync(peer, user, cultNetMessage).ConfigureAwait(false); return; } - var message = LiteNetLibTransportConnection.DecodeLegacy(frame); + Message message; + try + { + message = LiteNetLibTransportConnection.DecodeLegacy(frame); + } + catch (Exception ex) when (transport.WriteReceipts != null) + { + // A frame that decodes on neither channel is classed legacy, so this is where an unreadable frame lands. + // A receipt-capable peer is told which frame was unreadable (by its id, if the frame still holds one). + Logger.LogWarning($"CultNet frame unreadable: {ex.Message}"); + transport.AnswerUnreadableSchemaFrame(frame.Payload); + return; + } + if (LogSensitivePayloads) { Logger.LogDebug($"Received message: {MessagePackSerializer.ConvertToJson(new ReadOnlyMemory(frame.Payload))}"); @@ -519,7 +527,7 @@ namespace GameCult.Networking } }); - Logger.LogInfo($"Server started on port {ServerPort}."); + Logger.LogInfo($"Server started on port {Port}."); } private async Task HandleCultNetSchemaMessageAsync(NetPeer peer, User user, ICultNetSchemaMessage message) diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs new file mode 100644 index 00000000..8a1f0b46 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs @@ -0,0 +1,188 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using LiteNetLib; +using NUnit.Framework; +using static GameCult.Networking.Tests.NetworkingTests; + +namespace GameCult.Networking.Tests +{ + // The receipt server on a real socket: Server.Start on an ephemeral loopback port and real LiteNetLib clients. The + // handler-level tests (CultNetWriteReceiptServerTests) hand frames to the handlers; only a running server decides what + // a frame that does not decode is answered with. + public sealed class CultNetWriteReceiptLoopbackTests + { + private static readonly TimeSpan Cap = TimeSpan.FromSeconds(10); + private static readonly CultRecordKey KeyA = new("tests:loopback:a"); + private static readonly CultRecordKey KeyB = new("tests:loopback:b"); + private const string ShardId = "loopback-shard"; + + private sealed class Client : IDisposable + { + private readonly NetManager _manager; + private readonly NetPeer _peer; + private readonly BlockingCollection _received = new(); + public int Closed; + + public Client(int port, string key) + { + var listener = new EventBasedNetListener(); + var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var decoder = new LiteNetLibTransportConnection(); + listener.PeerConnectedEvent += _ => connected.TrySetResult(true); + listener.PeerDisconnectedEvent += (_, _) => Interlocked.Increment(ref Closed); + listener.NetworkReceiveEvent += (_, reader, _, _) => + _received.Add(LiteNetLibTransportConnection.DecodeSchema(decoder.Receive(reader.GetRemainingBytes()))); + _manager = new NetManager(listener) { UnsyncedEvents = true }; + _manager.Start(0); + _peer = _manager.Connect("127.0.0.1", port, key); + if (!connected.Task.Wait(Cap)) Assert.Fail("the client did not connect"); + } + + public void Send(T message) where T : ICultNetSchemaMessage => + _peer.Send(CultNetSchemaMessageSerialization.Serialize(message), DeliveryMethod.ReliableOrdered); + + public void SendRaw(byte[] bytes) => _peer.Send(bytes, DeliveryMethod.ReliableOrdered); + + public T Next() where T : ICultNetSchemaMessage + { + if (!_received.TryTake(out var message, Cap)) Assert.Fail($"no {typeof(T).Name} arrived within {Cap}"); + Assert.That(message, Is.InstanceOf()); + return (T)message!; + } + + public void ExpectNothingMore(string because) + { + // A reliable ordered channel: anything the server sent before the probe's answer has already arrived. + Assert.That(_received.Count, Is.Zero, because); + } + + public void Dispose() + { + _manager.Stop(); + _received.Dispose(); + } + } + + private sealed class Authority : ICultNetRemoteWriteAuthority + { + public CultNetRemoteWriteDecision Decide(CultNetRemoteWriteRequest request) => + request.RecordKey.Equals(KeyB) ? CultNetRemoteWriteDecision.Refuse("denied") : CultNetRemoteWriteDecision.Grant(); + } + + private sealed class Running : IDisposable + { + public Server Server = null!; + public CultCache Cache = null!; + public CultNetDatabase Database = null!; + public string SchemaId = string.Empty; + private CultNetDatabaseServer _databaseServer = null!; + + public static Running Start() + { + var cache = new CultCache(); + var schemaId = cache.Registry.GetRequired().SchemaId; + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: true, schemaIds: [schemaId])] + }); + var server = new Server(cache, ServerSecurityOptions.Development()) { Port = 0 }; + var databaseServer = new CultNetDatabaseServer(server, database, new CultNetDatabaseServerOptions + { + RemoteWriteAuthority = new Authority() + }); + server.Start(); + return new Running { Server = server, Cache = cache, Database = database, SchemaId = schemaId, _databaseServer = databaseServer }; + } + + public Client Connect() => new(Server.Port, ServerSecurityOptions.Development().ConnectionKey); + + public CultNetDocumentPutRawMessage PutOf(string id, CultRecordKey key, string text) + { + var message = Database.Documents.CreateRawDocumentPutMessage(id, new CultRecordHandle(key), + new NetworkSchemaNote { Schema = "tests.networking_note.v1", Text = text }); + message.ShardId = ShardId; + message.ShardEpoch = 1; + return message; + } + + public void Dispose() + { + _databaseServer.Dispose(); + Server.Dispose(); + } + } + + private static CultNetHelloMessage HelloOf(params string[] versions) => + new() { RuntimeId = "loopback-client", RuntimeKind = "test", SupportedMessageVersions = versions }; + + private static readonly string[] BothFrames = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1]; + + [Test] + public void StartBindsTheRequestedPortAndReportsTheOneBound() + { + using var running = Running.Start(); + Assert.That(running.Server.Port, Is.Not.Zero.And.Not.EqualTo(3075)); + } + + // contracts/cultnet/interop/write-receipt-vectors.json, error_v1_unreadable_frame_has_null_message_id: code + // message_invalid, details.field "message", messageId null. + [TestCase(new byte[] { 0xC1 }, TestName = "AnUnreadableFrameFromAReceiptPeerIsAnsweredWithMessageInvalidAndNoId_NeverUsedByte")] + [TestCase(new byte[] { 0x92, 0x01, 0x02 }, TestName = "AnUnreadableFrameFromAReceiptPeerIsAnsweredWithMessageInvalidAndNoId_NotAMap")] + public void AnUnreadableFrameFromAReceiptPeerIsAnsweredWithMessageInvalidAndNoId(byte[] garbage) + { + using var running = Running.Start(); + using var client = running.Connect(); + client.Send(HelloOf(BothFrames)); + client.Next(); + + client.SendRaw(garbage); + + var error = client.Next(); + Assert.That(error.Code, Is.EqualTo("message_invalid")); + Assert.That(error.Details!.Field, Is.EqualTo("message")); + Assert.That(error.MessageId, Is.Null); + } + + [Test] + public async Task AReceiptPeerIsStillServedAfterAnUnreadableFrameAndALegacyPeerIsAnsweredAsBefore() + { + using var running = Running.Start(); + using var receipt = running.Connect(); + receipt.Send(HelloOf(BothFrames)); + receipt.Next(); + receipt.SendRaw(new byte[] { 0xC1 }); + receipt.Next(); + + receipt.Send(running.PutOf("r1", KeyA, "one")); + Assert.That(receipt.Next().MessageId, Is.EqualTo("r1")); + + using var legacy = running.Connect(); + legacy.Send(HelloOf()); // a hello that lists neither frame is answered with nothing + legacy.SendRaw(new byte[] { 0xC1 }); // unanswered: were it answered, error.v1 would arrive ahead of the refusal + legacy.Send(running.PutOf("l1", KeyB, "two")); + var refusal = legacy.Next(); + Assert.That(refusal, Is.TypeOf()); + Assert.That(refusal.Code, Is.EqualTo("write_not_authorized")); + legacy.Send(running.PutOf("l2", KeyA, "three")); + await WaitFor(() => running.Cache.Get(KeyA)?.Text == "three"); + legacy.ExpectNothingMore("a legacy peer is never sent a receipt or error.v1"); + Assert.That(running.Cache.Get(KeyB), Is.Null); + } + + private static async Task WaitFor(Func condition) + { + var deadline = DateTime.UtcNow + Cap; + while (!condition()) + { + if (DateTime.UtcNow > deadline) Assert.Fail("the condition did not hold within the cap"); + await Task.Delay(5); + } + } + } +} From b89a8837003517ea0633fe46c63d2f6e517895bf Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:58:51 +0000 Subject: [PATCH 037/117] spec: C# RUDP schema server is not receipt-capable; advertise only if end to end Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 4b97468f..3d7dd19b 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -18,7 +18,7 @@ Why a new error version and not an extra key on v0: the v0 schema is closed (`ad ## Capability: opting in -A connection is receipt-capable when its `cultnet.hello.v0` (sent before its first write; the schema channel is reliable and ordered) lists both `cultnet.write_admitted.v0` and `cultnet.error.v1` in `supportedMessageVersions`. A server MUST NOT send either new frame to any other connection, and answers it exactly as before (no receipt, `error.v0`). A receipt-capable server lists both in its own hello, so a client can tell a legacy server from a receipt server that has not answered yet. A client that sends no hello gets legacy behaviour. +A connection is receipt-capable when its `cultnet.hello.v0` (sent before its first write; the schema channel is reliable and ordered) lists both `cultnet.write_admitted.v0` and `cultnet.error.v1` in `supportedMessageVersions`. A server MUST NOT send either new frame to any other connection, and answers it exactly as before (no receipt, `error.v0`). A receipt-capable server lists both in its own hello, so a client can tell a legacy server from a receipt server that has not answered yet. A client that sends no hello gets legacy behaviour. A server advertises the two frames in its hello only if it implements the receipt rules end to end. **The capability changes framing only; it never authorizes.** A hello authenticates nothing and grants nothing: who may write, and which keys, is decided by the server from the transport-authenticated actor exactly as without the capability. Declaring the capability never makes a refused write admitted, never widens an actor's authority, and never changes what a write does; it selects only which frame the answer arrives in. @@ -61,6 +61,14 @@ Those clients complete *successfully* when the window ends with no refusal. Unde `write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +## Conformance status + +| Implementation | Receipt-capable | Hello advertises the frames | +|---|---|---| +| C# RUDP schema server (`src/GameCult.Networking/CultNetRudpSchemaServer.cs`) | no | no | + +It is not receipt-capable and must not advertise the frames until it implements the rules end to end. + ## Conformance follow-ups (not in this PR) C# server and client, Rust, TypeScript, Python, Kotlin: emit and parse the two frames; hello capability; tri-state client outcome; replace the #55 to #58 window-success; re-capture the wire vectors in each runtime. Server write paths include the shard write forwarder (rule 6) and the Odin catalog. From 541d734675b430d8729c9e883b0162961c47f6a9 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:02:56 +0000 Subject: [PATCH 038/117] C#: a forward is retryable only when provably not sent; unconfirmed otherwise; refusals carry fixed text CultNetWriteNotSentException names the pre-send boundary (connect, hello, no endpoint, or a send port that proves nothing was written). Every other forwarding failure keeps the session claim. ForUnownedSchema and the routing refusal no longer carry exception text. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetDatabaseServer.cs | 13 +- .../CultNetSchemaClient.cs | 3 +- .../CultNetSchemaMessages.cs | 2 +- .../CultNetShardWriteForwarding.cs | 56 ++++-- .../CultNetWriteCompletion.cs | 18 ++ .../CultNetWriteReceipts.cs | 8 +- .../CultNetWriteReceiptServerTests.cs | 176 +++++++++++++++++- 7 files changed, 244 insertions(+), 32 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index a6e55c3d..efe23be2 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -407,10 +407,10 @@ namespace GameCult.Networking } _server.Logger.LogWarning($"CultNet forwarded write was not answered by its primary: {failure.Message}"); - // Silence past the window means the primary may have applied it: never forward the id again on this session. - // Any other failure never reached a primary that could have. - if (failure is CultNetWriteUnconfirmedException) write.LeftSilent(); - else write.Unanswered(); + // Only a forward that provably never left (not sent) may be forwarded again. Every other failure, silence or not, is + // unconfirmed: the primary may have applied it, so the id is never forwarded again on this session. + if (failure is CultNetWriteNotSentException) write.NotSent(); + else write.LeftSilent(); } private async Task HandlePutAsync(CultNetDocumentPutRawMessage message, CultNetServerPeer peer) @@ -817,11 +817,14 @@ namespace GameCult.Networking exception is CultNetShardLogException || exception is AggregateException aggregate && aggregate.Flatten().InnerExceptions.Any(inner => inner is CultNetShardLogException); + // Fixed text: the routing refusal never carries an exception's own message (the shard and reason travel in the hint). + internal const string RoutingRefusalText = "the write belongs to another shard primary"; + private static CultNetErrorMessage CreateRoutingError(CultNetShardAuthorityException exception) { return new CultNetErrorMessage { - Error = exception.Message, + Error = RoutingRefusalText, RoutingHint = new CultNetShardRoutingHint { Reason = exception.Reason, diff --git a/src/GameCult.Networking/CultNetSchemaClient.cs b/src/GameCult.Networking/CultNetSchemaClient.cs index 83865e31..79f0ce20 100644 --- a/src/GameCult.Networking/CultNetSchemaClient.cs +++ b/src/GameCult.Networking/CultNetSchemaClient.cs @@ -27,7 +27,8 @@ namespace GameCult.Networking /// /// Sends one schema-v0 message through the transport. Fire-and-forget: an ack or a return is not admission; - /// read the result frame (). + /// read the result frame (). Throws only + /// when it can prove no byte was written; any other exception leaves the send ambiguous and a forwarded write unconfirmed. /// void SendCultNet(T message) where T : ICultNetSchemaMessage; diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 350ef43b..090c6a45 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -546,7 +546,7 @@ namespace GameCult.Networking public static CultNetErrorMessage ForUnownedSchema(CultNetUnownedSchemaException ex) => new CultNetErrorMessage { - Error = $"unowned_schema: {ex.Message}", + Error = "unowned_schema: no shard owns the schema and key", Code = "unowned_schema", Details = new CultNetErrorDetails { Field = "schemaId", Value = ex.SchemaId } }; diff --git a/src/GameCult.Networking/CultNetShardWriteForwarding.cs b/src/GameCult.Networking/CultNetShardWriteForwarding.cs index cb390308..e4e66298 100644 --- a/src/GameCult.Networking/CultNetShardWriteForwarding.cs +++ b/src/GameCult.Networking/CultNetShardWriteForwarding.cs @@ -11,8 +11,9 @@ namespace GameCult.Networking { /// /// Forwards a raw document put to the shard owner. Completes only when the owner answered that it admitted the write; - /// throws when it refused and - /// when it answered nothing. The server relays to the original peer exactly what the owner answered. + /// throws when it refused, when + /// nothing could have reached it (the only failure after which the id may be forwarded again), and any other exception + /// when the frame may have gone out and was not answered (unconfirmed, including ). The server relays to the original peer exactly what the owner answered. /// Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message); @@ -103,10 +104,10 @@ namespace GameCult.Networking public async Task ForwardPutAsync(CultNetShardDescriptor shard, CultNetDocumentPutRawMessage message) { if (message == null) throw new ArgumentNullException(nameof(message)); - var endpoint = ResolvePrimaryEndpoint(shard); + if (shard == null) throw new ArgumentNullException(nameof(shard)); message.ShardId ??= shard.ShardId; message.ShardEpoch ??= shard.Epoch; - await SendAsync(endpoint, message, message.MessageId).ConfigureAwait(false); + await SendAsync(shard, message, message.MessageId).ConfigureAwait(false); } /// @@ -116,10 +117,10 @@ namespace GameCult.Networking public async Task ForwardDeleteAsync(CultNetShardDescriptor shard, CultNetDocumentDeleteMessage message) { if (message == null) throw new ArgumentNullException(nameof(message)); - var endpoint = ResolvePrimaryEndpoint(shard); + if (shard == null) throw new ArgumentNullException(nameof(shard)); message.ShardId ??= shard.ShardId; message.ShardEpoch ??= shard.Epoch; - await SendAsync(endpoint, message, message.MessageId).ConfigureAwait(false); + await SendAsync(shard, message, message.MessageId).ConfigureAwait(false); } /// @@ -143,19 +144,40 @@ namespace GameCult.Networking return (uri.Host, uri.IsDefaultPort ? 3075 : uri.Port); } - private async Task SendAsync(string endpoint, T message, string messageId) where T : ICultNetSchemaMessage + // The write frame has not been handed to the transport until this returns: any failure here is CultNetWriteNotSentException, + // the one failure the caller may retry under the same id. + private async Task ConnectBeforeSendAsync(CultNetShardDescriptor shard, string messageId) { - var (host, port) = ParseEndpoint(endpoint); - using var client = CreateClient(endpoint); - client.Connect(host, port); - await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false); - // The hello comes first on the ordered schema channel: it opts this connection in to the primary's receipts. - client.SendCultNet(new CultNetHelloMessage + ICultNetSchemaClient? client = null; + try { - RuntimeId = "gamecult-networking-shard-forwarder", - RuntimeKind = "dotnet", - SupportedMessageVersions = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1] - }); + var endpoint = ResolvePrimaryEndpoint(shard); + var (host, port) = ParseEndpoint(endpoint); + client = CreateClient(endpoint); + client.Connect(host, port); + await WaitForConnectionAsync(client, endpoint).ConfigureAwait(false); + // The hello comes first on the ordered schema channel: it opts this connection in to the primary's receipts. + client.SendCultNet(new CultNetHelloMessage + { + RuntimeId = "gamecult-networking-shard-forwarder", + RuntimeKind = "dotnet", + SupportedMessageVersions = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1] + }); + return client; + } + catch (Exception failure) + { + client?.Dispose(); + if (failure is CultNetWriteNotSentException) throw; + throw new CultNetWriteNotSentException(messageId, failure); + } + } + + private async Task SendAsync(CultNetShardDescriptor shard, T message, string messageId) where T : ICultNetSchemaMessage + { + using var client = await ConnectBeforeSendAsync(shard, messageId).ConfigureAwait(false); + // From the write's send on, every failure (timeout, reset, silence, an exception from the send itself) propagates as it + // is: the frame may have gone out, so the caller treats it as unconfirmed. await CultNetWriteCompletion.SendAndAwaitReceiptAsync( (admitted, refuse, refuseV1) => { diff --git a/src/GameCult.Networking/CultNetWriteCompletion.cs b/src/GameCult.Networking/CultNetWriteCompletion.cs index be8a4632..8767be6e 100644 --- a/src/GameCult.Networking/CultNetWriteCompletion.cs +++ b/src/GameCult.Networking/CultNetWriteCompletion.cs @@ -40,6 +40,24 @@ namespace GameCult.Networking public string MessageId { get; } } + /// + /// A forwarded write that provably never left this process: the connection failed, or the send failed before any byte was + /// written. It is the only forwarding failure after which the same id may be forwarded again. A send port that cannot prove + /// nothing was written must not throw this; every other failure of a forward is unconfirmed. + /// + public sealed class CultNetWriteNotSentException : Exception + { + /// Creates the exception for the write ; is for logs only. + public CultNetWriteNotSentException(string messageId, Exception? cause = null) + : base($"Write '{messageId}' was not sent.", cause) + { + MessageId = messageId; + } + + /// Gets the id of the write that was not sent. + public string MessageId { get; } + } + /// /// Completes a write on the peer's result frame, never on a send's return or a transport acknowledgement: /// an ack proves delivery, not admission. diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index 513efd99..cb13eeae 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -245,8 +245,8 @@ namespace GameCult.Networking } /// - /// A forward its primary left unanswered: no answer is owed or possible, and the claim stays so the id is never - /// forwarded again on this session. Sends nothing. + /// A forward that may have reached its primary and is unconfirmed (silence, reset, timeout, an ambiguous send): no answer is + /// owed or possible, and the claim stays so the id is never forwarded again on this session. Sends nothing. /// public void LeftSilent() { @@ -254,8 +254,8 @@ namespace GameCult.Networking _answered = true; } - /// A forward that never reached a primary that could have applied it: forget the claim, send nothing. - public void Unanswered() + /// A forward that was provably not sent (connect failed, or the send failed before any byte was written): forget the claim, send nothing. + public void NotSent() { if (_answered) return; _answered = true; diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index b93b43b3..cab649e2 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -636,10 +636,9 @@ namespace GameCult.Networking.Tests } [Test] - public async Task AForwardThatNeverReachedThePrimaryMayBeRetried() + public async Task AForwardThatWasNotSentMayBeRetriedUnderTheSameId() { - Exception failure = new TimeoutException("primary unreachable"); - var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(failure) }; + var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(new CultNetWriteNotSentException("fwd-1", new TimeoutException("primary unreachable"))) }; var rig = Build(primary: false, forwarder: forwarder); var wire = new Wire(); await rig.Hello(wire, BothFrames); @@ -651,7 +650,176 @@ namespace GameCult.Networking.Tests forwarder.OnPut = _ => Task.CompletedTask; await rig.Put(wire, "fwd-1", KeyA, "one"); AssertAdmitted(wire.Sent.Single(), "fwd-1", "put", KeyA.Value); - Assert.That(forwarder.Forwarded, Is.EqualTo(2), "the unanswered attempt left no claim behind"); + Assert.That(forwarder.Forwarded, Is.EqualTo(2), "the unsent attempt left no claim behind"); + } + + // Once the frame may have gone out, whatever failed, the id is never forwarded again on the session. + private static IEnumerable PostSendFailures() + { + yield return new TestCaseData(new TimeoutException("no answer")).SetName("timeout"); + yield return new TestCaseData(new System.IO.IOException("connection reset")).SetName("reset"); + yield return new TestCaseData(new InvalidOperationException("client disconnected")).SetName("disconnected"); + yield return new TestCaseData(new CultNetWriteUnconfirmedException("fwd-1")).SetName("no answer in the window"); + yield return new TestCaseData(new Exception("anything else, ambiguous")).SetName("ambiguous"); + } + + [TestCaseSource(nameof(PostSendFailures))] + public async Task AForwardThatMayHaveBeenSentIsNeverForwardedAgainOnTheSession(Exception failure) + { + var forwarder = new ScriptedForwarder { OnPut = _ => Task.FromException(failure) }; + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + await rig.Put(wire, "fwd-1", KeyA, "one"); + forwarder.OnPut = _ => Task.CompletedTask; + await rig.Put(wire, "fwd-1", KeyA, "one"); + + Assert.That(wire.Sent, Is.Empty, "unconfirmed: nothing answered, nothing synthesized"); + Assert.That(forwarder.Forwarded, Is.EqualTo(1), "the claim stays"); + } + + [Test] + public async Task TheRealForwarderRetriesAConnectFailureButNotAFailureAfterTheWriteWasSent() + { + var clients = new Queue(); + var forwarder = new CultNetSchemaWriteForwarder(new CultNetSchemaWriteForwarderOptions + { + CreateClient = () => clients.Dequeue(), + ConnectTimeout = TimeSpan.FromMilliseconds(100), + ResultWindow = TimeSpan.FromMilliseconds(100) + }); + var rig = Build(primary: false, forwarder: forwarder); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + + clients.Enqueue(new ForwardingClient { ConnectFails = true }); + await rig.Put(wire, "fwd-1", KeyA, "one"); + var sent = new ForwardingClient { AdmitWrites = true }; + clients.Enqueue(sent); + await rig.Put(wire, "fwd-1", KeyA, "one"); + AssertAdmitted(wire.Sent.Single(), "fwd-1", "put", KeyA.Value); + + wire.Sent.Clear(); + var reset = new ForwardingClient { WriteSendFails = true }; + clients.Enqueue(reset); + await rig.Put(wire, "fwd-2", KeyA, "two"); + clients.Enqueue(new ForwardingClient { AdmitWrites = true }); + await rig.Put(wire, "fwd-2", KeyA, "two"); + Assert.That(wire.Sent, Is.Empty, "the write send threw: it may have gone out"); + Assert.That(clients, Has.Count.EqualTo(1), "and the id was not forwarded again"); + } + + private sealed class ForwardingClient : ICultNetSchemaClient + { + private Action? _admitted; + public bool ConnectFails { get; set; } + public bool NeverConnects { get; set; } + public bool HelloSendFails { get; set; } + public bool WriteSendFails { get; set; } + public bool WriteNotSentFails { get; set; } + public bool AdmitWrites { get; set; } + public bool Connected { get; private set; } + public List Sent { get; } = new(); + + public void Connect(string host, int port) + { + if (ConnectFails) throw new System.Net.Sockets.SocketException(); + Connected = !NeverConnects; + } + + public void SendCultNet(T message) where T : ICultNetSchemaMessage + { + if (message is CultNetHelloMessage) + { + if (HelloSendFails) throw new System.IO.IOException("hello failed"); + return; + } + + if (WriteNotSentFails) throw new CultNetWriteNotSentException("x"); + if (WriteSendFails) throw new System.IO.IOException("send failed after an unknown number of bytes"); + Sent.Add(message); + if (AdmitWrites && message is CultNetDocumentPutRawMessage put) + _admitted?.Invoke(new CultNetWriteAdmittedMessage { MessageId = put.MessageId, Operation = "put", SchemaId = put.Document?.SchemaId, RecordKey = put.Document?.RecordKey }); + } + + public void OnCultNet(Action callback) where T : ICultNetSchemaMessage + { + if (callback is Action admitted) _admitted = admitted; + } + + public void Dispose() { } + } + + private static CultNetSchemaWriteForwarder RealForwarder(ForwardingClient client) => new(new CultNetSchemaWriteForwarderOptions + { + CreateClient = () => client, + ConnectTimeout = TimeSpan.FromMilliseconds(100), + ResultWindow = TimeSpan.FromMilliseconds(100) + }); + + private static readonly CultNetShardDescriptor ForwardShard = new("notes", "primary", 1, isPrimary: false, primaryEndpoints: ["cultnet://primary.example.test:4075"]); + + private static CultNetDocumentPutRawMessage ForwardPut() => new() { MessageId = "fwd-1", Document = new CultNetRawDocumentRecord { SchemaId = "s", RecordKey = "k", Payload = [1] } }; + + [TestCase("connect")] + [TestCase("never connects")] + [TestCase("hello")] + [TestCase("write not sent")] + public void TheForwarderNamesEveryFailureBeforeTheWriteWasSentAsNotSent(string stage) + { + var client = new ForwardingClient + { + ConnectFails = stage == "connect", + NeverConnects = stage == "never connects", + HelloSendFails = stage == "hello", + WriteNotSentFails = stage == "write not sent" + }; + + Assert.ThrowsAsync(() => RealForwarder(client).ForwardPutAsync(ForwardShard, ForwardPut())); + Assert.That(client.Sent, Is.Empty); + } + + [Test] + public void TheForwarderNamesNoPrimaryEndpointAsNotSent() + { + var shard = new CultNetShardDescriptor("notes", "primary", 1, isPrimary: false, primaryEndpoints: []); + Assert.ThrowsAsync(() => RealForwarder(new ForwardingClient()).ForwardPutAsync(shard, ForwardPut())); + } + + [Test] + public void ASendThatThrewWithoutProvingNothingWasWrittenIsNotNotSent() + { + var client = new ForwardingClient { WriteSendFails = true }; + var failure = Assert.CatchAsync(() => RealForwarder(client).ForwardPutAsync(ForwardShard, ForwardPut())); + Assert.That(failure, Is.Not.InstanceOf(), "ambiguous: unconfirmed"); + } + + [Test] + public void ASentWriteNobodyAnswersIsUnconfirmedNotNotSent() + { + var client = new ForwardingClient(); + Assert.ThrowsAsync(() => RealForwarder(client).ForwardPutAsync(ForwardShard, ForwardPut())); + Assert.That(client.Sent, Has.Count.EqualTo(1)); + } + + [Test] + public async Task TheRoutingRefusalAndTheUnownedSchemaRefusalCarryNoExceptionText() + { + var rig = Build(primary: false); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + await rig.Put(wire, "fwd-1", KeyA, "one"); + var routing = AssertRefused(wire.Sent.Single(), "fwd-1", null); + Assert.That(routing.Error, Is.EqualTo(CultNetDatabaseServer.RoutingRefusalText)); + + var secret = new CultNetUnownedSchemaException("secret.schema.v1", new CultRecordKey("secret:key")); + var unowned = CultNetErrorMessage.ForUnownedSchema(secret); + Assert.That(unowned.Error, Does.Not.Contain(secret.Message).And.Not.Contain("secret")); + Assert.That(unowned.Details!.Value, Is.EqualTo("secret.schema.v1"), "the structured detail still names the schema"); } [Test] From 734087766ae84a407367239c960ea80cdb151a3f Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:04:21 +0000 Subject: [PATCH 039/117] Spec #59: id size, digest memory, limit refusals remembered, required fields, reliable channel, local send failure, per-class conformance table Co-Authored-By: Claude Sonnet 5.5 --- .../interop/cultnet.error.v1.schema.json | 4 ++-- contracts/cultnet/interop/write-receipt.md | 23 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/contracts/cultnet/interop/cultnet.error.v1.schema.json b/contracts/cultnet/interop/cultnet.error.v1.schema.json index cd9f47b5..02a1ae21 100644 --- a/contracts/cultnet/interop/cultnet.error.v1.schema.json +++ b/contracts/cultnet/interop/cultnet.error.v1.schema.json @@ -45,7 +45,7 @@ } }, "code": { - "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each). limit_reached: a receipt-capable server refuses a NEW write because its per-session remembered-write table (1024 entries) is full; messageId is the write's id.", + "description": "Machine-readable refusal code (R-N). Additive: a v0 peer that does not know this field ignores it. The set is a closed enum: Rust and TypeScript refuse a message whose code is not listed, so adding a code is one coordinated change across every runtime (this schema, the C# factory, the Rust CultNetErrorCode, and a captured-bytes case for each). limit_reached: a receipt-capable server refuses a NEW write because its per-session remembered-write table (1024 entries) or its server-wide memory budget is full; messageId is the write's id.", "type": [ "string", "null" @@ -79,7 +79,7 @@ ], "properties": { "field": { - "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\". message_invalid: the request field that is missing or ill-typed (\"message\" when the whole frame cannot be read). limit_reached: no field; details stay all null.", + "description": "selection_invalid: the selection field that failed. variant_unsupported: \"subscriptionId\", when a subscription was ended. unowned_schema: \"schemaId\". write_not_authorized: \"recordKey\". unsupported_schema_version: \"schemaVersion\". message_invalid: the request field that is missing or ill-typed (\"message\" when the whole frame cannot be read). limit_reached: no field; details stay all null. message_invalid also names \"messageId\" (over 256 bytes), \"document\", \"schemaId\" or \"channel\" (write on an unreliable channel).", "type": [ "string", "null" diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 3d7dd19b..5a7b90f0 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -28,11 +28,17 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir 2. **After the commit point.** `write_admitted` is sent only after the write is committed by the same rule that makes it visible to readers. Receipt of the bytes, parse, or authorisation alone is not admission. 3. **Id on every refusal.** Every refusal of a write, whatever its code (`write_not_authorized`, `message_invalid`, `unowned_schema`, `unsupported_schema_version`, shard authority errors, ...), is an `error.v1` whose `messageId` is the write's id. When the frame is readable enough to find the id, it is found; `message_invalid` for a missing or ill-typed field still carries the id. Nil only when no id can be read. 4. **Batches.** There is no batch frame; a batch is N writes, each with its own `messageId`, each answered under rules 1 to 3. A failed batch names its member by that id. A server may not collapse a batch into one answer. -5. **Identity and idempotent resend.** A `messageId` is unique per authenticated actor and session: the client mints it, never reuses it for different content in that session, and a server never accepts the id of one actor as the id of another. The server's dedupe key is `(actor, messageId)`, where *actor* is the transport-authenticated identity, never a name the hello or the frame claims. Alongside the key it keeps the **content hash** of the write and the answer it gave. The content hash is SHA-256 of the MessagePack array `[operation, schemaId, recordKey, payload]`, where `payload` is the raw document payload as a MessagePack `bin` for a put and nil for a delete (the vectors give worked hashes). **Scope: at least the session.** The key lives for the life of the session it was made in, and a server may bound how many writes it remembers per session (a session at the bound refuses new writes with an `error.v1` carrying the new write's id, and keeps answering the ids it remembers; it never forgets one it promised to answer). A server MAY keep the key longer for the same actor, up to `DEDUPE_RETENTION_MS` (10 minutes), but a client MUST NOT rely on it: **a reconnect is a new session, and a resend on the new session is a new write.** +5. **Identity and idempotent resend.** A `messageId` is unique per authenticated actor and session: the client mints it, never reuses it for different content in that session, and a server never accepts the id of one actor as the id of another. The server's dedupe key is `(actor, messageId)`, where *actor* is the transport-authenticated identity, never a name the hello or the frame claims. Alongside the key it keeps the **content hash** of the write and the answer it gave. The content hash is SHA-256 of the MessagePack array `[operation, schemaId, recordKey, payload]`, where `payload` is the raw document payload as a MessagePack `bin` for a put and nil for a delete (the vectors give worked hashes). **Scope: at least the session.** The key lives for the life of the session it was made in, and a server may bound how many writes it remembers per session (see Remembered-write limits; a server never forgets an id it promised to answer). A server MAY keep the key longer for the same actor, up to `DEDUPE_RETENTION_MS` (10 minutes), but a client MUST NOT rely on it: **a reconnect is a new session, and a resend on the new session is a new write.** - Same key, same content hash, answer was `write_admitted`: answer `write_admitted` again with the same body. The write is not applied twice. - Same key, same content hash, answer was a refusal: the same refusal again. - **Same key, different content hash: refuse `message_invalid` (details `field: "messageId"`) and never re-send the old receipt.** Old receipts are for the old content; showing one for new content would report an unapplied write as admitted. The original write's state is unchanged. - - **Remembered-write limit.** The per-session table of remembered writes holds at most 1024 entries. A server whose table is full refuses a NEW write (a key not already in the table) with `limit_reached`, `messageId` the write's id, `details` all null (no field is named); the write is not applied. A resend of a remembered key is still answered from the table. + - **Id size.** A write's `messageId` is at most 256 bytes of UTF-8. A longer id is refused `message_invalid` (`details.field: "messageId"`, the refusal's `messageId` the id as received, text fixed per Fixed texts) BEFORE anything is remembered: no table entry, no digest, no budget charge. Such a refusal is therefore not replayed from the table; a resend is refused afresh, identically. + - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. + - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. + - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and its resend is refused identically). The client mints a new id on a new session. + - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. + - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. + - **Reliable channel only.** A write arriving on an unreliable channel on a receipt session is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. @@ -43,7 +49,7 @@ A write settles as exactly one of: - **admitted**: a `write_admitted` whose `messageId` is the write's. - **refused**: an `error.v1` whose `messageId` is the write's, carrying its `code` and `details` as the typed error. -- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window) and `session_ended` (the session ended first). Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. +- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window) and `session_ended` (the session ended first, whether the peer closed it or the client's own send failed: a local send failure carries a local-failure reason, e.g. `session_ended` / `local_send_failed`). A local send failure is never reported as refused: no server answer exists, and the write may or may not have left the machine. Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. **Result window.** Two named constants (also in the vectors): `RECEIPT_RESULT_WINDOW_MS = 5000`, the default against a receipt-capable server (its hello lists both frames); `LEGACY_RESULT_WINDOW_MS = 500`, used only as the fallback against a server whose hello does not list them (or that sent none). The window runs from the send. A receipt-capable server that has not answered within its window is reported as unconfirmed like any other; clients log it as a server that missed its window. @@ -63,11 +69,14 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Conformance status -| Implementation | Receipt-capable | Hello advertises the frames | -|---|---|---| -| C# RUDP schema server (`src/GameCult.Networking/CultNetRudpSchemaServer.cs`) | no | no | +| Server class | Advertises both frames | +|---|---| +| `CultNetDatabaseServer` | yes | +| Shard write forwarder | yes | +| Rust RUDP document server | yes | +| C# RUDP schema server (`CultNetRudpSchemaServer`, `src/GameCult.Networking/CultNetRudpSchemaServer.cs`) | no | -It is not receipt-capable and must not advertise the frames until it implements the rules end to end. +Advertising is per server class, not per transport. A class advertises both frames in its hello only if it implements the receipt rules end to end; the C# RUDP schema server stays "no" until it answers writes with receipts. ## Conformance follow-ups (not in this PR) From c4d1efe000928f28ac3a223b96993e1a5d05be48 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:07:23 +0000 Subject: [PATCH 040/117] C#: receipt ids capped at 256 bytes and held as digests; limit refusals remembered, no slot freed; required fields refused with fixed text; RUDP server advertises no receipt frames Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetDatabaseServer.cs | 23 ++- .../CultNetWriteReceipts.cs | 150 ++++++++++---- .../CultNetRudpSchemaServerPeerInputTests.cs | 40 ++++ .../CultNetWriteReceiptServerTests.cs | 187 ++++++++++++++++++ 4 files changed, 360 insertions(+), 40 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index efe23be2..32688434 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -416,9 +416,17 @@ namespace GameCult.Networking private async Task HandlePutAsync(CultNetDocumentPutRawMessage message, CultNetServerPeer peer) { var write = BeginReceipted(peer, "put", message.MessageId, message.Document?.SchemaId, message.Document?.RecordKey, message.Document?.Payload); - if (write != null && !write.HasId) + var idRefusal = write?.IdRefusal; + if (idRefusal != null) { - write.Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "a write needs a messageId")); + write!.Refused(idRefusal); + return; + } + + var missing = message.Document == null ? "document" : message.Document.SchemaId == null ? "schemaId" : null; + if (missing != null) + { + Refuse(peer, write, CultNetErrorMessage.ForMessageInvalid(missing, $"{missing} is required")); return; } @@ -484,9 +492,16 @@ namespace GameCult.Networking private async Task HandleDeleteAsync(CultNetDocumentDeleteMessage message, CultNetServerPeer peer) { var write = BeginReceipted(peer, "delete", message.MessageId, message.SchemaId, message.RecordKey, null); - if (write != null && !write.HasId) + var idRefusal = write?.IdRefusal; + if (idRefusal != null) { - write.Refused(CultNetErrorMessage.ForMessageInvalid("messageId", "a write needs a messageId")); + write!.Refused(idRefusal); + return; + } + + if (message.SchemaId == null) + { + Refuse(peer, write, CultNetErrorMessage.ForMessageInvalid("schemaId", "schemaId is required")); return; } diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index cb13eeae..7e984287 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -47,73 +47,138 @@ namespace GameCult.Networking } } + /// A SHA-256 digest held by value: the fixed-size stand-in for an id or a content hash. + internal readonly struct CultNetDigest : IEquatable + { + private readonly ulong _a, _b, _c, _d; + + private CultNetDigest(ulong a, ulong b, ulong c, ulong d) { _a = a; _b = b; _c = c; _d = d; } + + public static CultNetDigest Of(byte[] bytes) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(bytes); + return new CultNetDigest( + BitConverter.ToUInt64(hash, 0), BitConverter.ToUInt64(hash, 8), + BitConverter.ToUInt64(hash, 16), BitConverter.ToUInt64(hash, 24)); + } + + public bool Equals(CultNetDigest other) => _a == other._a && _b == other._b && _c == other._c && _d == other._d; + public override bool Equals(object? obj) => obj is CultNetDigest other && Equals(other); + public override int GetHashCode() => unchecked((int)_a ^ (int)(_a >> 32)); + public static bool operator ==(CultNetDigest left, CultNetDigest right) => left.Equals(right); + public static bool operator !=(CultNetDigest left, CultNetDigest right) => !left.Equals(right); + } + /// /// One receipt-capable peer's writes: message id to content hash and the answer given, refusals included, so a resend of the - /// same content is answered again without being decided again and the same id with other content is refused. + /// same content is answered again without being decided again and the same id with other content is refused. Ids and content + /// hashes are held as fixed-size digests; the answer is held as its kind (and, for a refusal, its code, text and details), + /// never the strings of the write. /// internal sealed class CultNetWriteReceiptSession { /// The most writes a peer is remembered for, refused ones included; a peer at the limit has new writes refused. internal const int MaxRememberedWrites = 1024; + /// The most ids refused at the limit that are remembered, so their resend is judged the same way. + internal const int MaxRememberedLimitRefusals = 1024; + + /// The longest messageId a write may carry, in UTF-8 bytes (write-receipt.md, Id size). + internal const int MaxMessageIdBytes = 256; + /// What a write is refused with when it failed for a reason no typed refusal names. Never an exception's own text. internal const string WriteFailedText = "the write could not be applied"; private readonly object _gate = new(); - private readonly Dictionary _writes = new(StringComparer.Ordinal); + private readonly Dictionary _writes = new(); + private readonly Dictionary _limitRefusals = new(); private sealed class Entry { - public Entry(byte[] hash) { Hash = hash; } - public byte[] Hash { get; } - public ICultNetSchemaMessage? Answer { get; set; } + public Entry(CultNetDigest hash) { Hash = hash; } + public CultNetDigest Hash { get; set; } + public bool Abandoned { get; set; } + public bool Admitted { get; set; } + public CultNetErrorMessage? Refusal { get; set; } } - public enum Begin { New, Replay, Unanswered, DifferentContent, Full } + public enum Begin { New, ReplayAdmitted, ReplayRefusal, Unanswered, DifferentContent, Full } + + private static CultNetDigest IdDigest(string messageId) => CultNetDigest.Of(System.Text.Encoding.UTF8.GetBytes(messageId)); /// /// Looks up first. An id not seen is claimed for this content (); - /// otherwise the outcome says what already holds it, and is the answer to replay. + /// otherwise the outcome says what already holds it, and is the refusal to repeat. /// - public Begin TryBegin(string messageId, byte[] hash, out ICultNetSchemaMessage? answer) + public Begin TryBegin(string messageId, byte[] hash, out CultNetErrorMessage? refusal) { + var id = IdDigest(messageId); + var content = CultNetDigest.Of(hash); lock (_gate) { - answer = null; - if (_writes.TryGetValue(messageId, out var entry)) + refusal = null; + if (_writes.TryGetValue(id, out var entry)) { - if (!CryptographicOperations.FixedTimeEquals(entry.Hash, hash)) return Begin.DifferentContent; - if (entry.Answer != null) + if (entry.Abandoned) { - answer = entry.Answer; - return Begin.Replay; + // Not sent, so judged afresh in the slot it already holds; no slot is ever freed. + entry.Abandoned = false; + entry.Hash = content; + return Begin.New; + } + + if (entry.Hash != content) return Begin.DifferentContent; + if (entry.Admitted) return Begin.ReplayAdmitted; + if (entry.Refusal != null) + { + refusal = entry.Refusal; + return Begin.ReplayRefusal; } // No answer yet: the first attempt is still being applied, or its primary left it unanswered for good. return Begin.Unanswered; } - if (_writes.Count >= MaxRememberedWrites) return Begin.Full; - _writes[messageId] = new Entry(hash); + if (_writes.Count >= MaxRememberedWrites) + { + if (_limitRefusals.TryGetValue(id, out var refused)) return refused == content ? Begin.Full : Begin.DifferentContent; + if (_limitRefusals.Count < MaxRememberedLimitRefusals) _limitRefusals[id] = content; + return Begin.Full; + } + + _writes[id] = new Entry(content); return Begin.New; } } - /// The write was answered, admitted or refused: remember the answer so a resend repeats it. - public void Answered(string messageId, ICultNetSchemaMessage answer) + /// The write was admitted: remember it so a resend repeats the receipt. + public void Admitted(string messageId) { lock (_gate) { - if (_writes.TryGetValue(messageId, out var entry)) entry.Answer = answer; + if (_writes.TryGetValue(IdDigest(messageId), out var entry)) entry.Admitted = true; } } - /// The write never reached anyone who could have applied it: forget it so a repeat is judged afresh. + /// The write was refused: remember the refusal so a resend repeats it. + public void Refused(string messageId, CultNetErrorMessage refusal) + { + lock (_gate) + { + if (_writes.TryGetValue(IdDigest(messageId), out var entry)) entry.Refusal = refusal; + } + } + + /// + /// The write never reached anyone who could have applied it: a repeat is judged afresh. The entry keeps its slot, so + /// forgetting a write never makes room for an id that was refused at the limit. + /// public void Abandoned(string messageId) { lock (_gate) { - _writes.Remove(messageId); + if (_writes.TryGetValue(IdDigest(messageId), out var entry)) entry.Abandoned = true; } } } @@ -193,15 +258,19 @@ namespace GameCult.Networking /// public bool Begin() { - var outcome = _session.TryBegin(_messageId, CultNetWriteContentHash.Compute(_operation, _schemaId, _recordKey, _payload), out var answer); + var outcome = _session.TryBegin(_messageId, CultNetWriteContentHash.Compute(_operation, _schemaId, _recordKey, _payload), out var refusal); switch (outcome) { case CultNetWriteReceiptSession.Begin.New: _claimed = true; return true; - case CultNetWriteReceiptSession.Begin.Replay: + case CultNetWriteReceiptSession.Begin.ReplayAdmitted: _answered = true; - _send(answer!); + _send(AdmittedReceipt()); + return false; + case CultNetWriteReceiptSession.Begin.ReplayRefusal: + _answered = true; + _send(CultNetErrorV1Message.From(refusal!, _messageId)); return false; case CultNetWriteReceiptSession.Begin.Unanswered: // The first attempt is still being applied and will answer this id, or its primary left it unanswered: @@ -218,20 +287,21 @@ namespace GameCult.Networking } } + private CultNetWriteAdmittedMessage AdmittedReceipt() => new() + { + MessageId = _messageId, + Operation = _operation, + SchemaId = _schemaId, + RecordKey = _recordKey + }; + /// The write committed: remember the receipt and send it. public void Admitted() { if (_answered) return; _answered = true; - var receipt = new CultNetWriteAdmittedMessage - { - MessageId = _messageId, - Operation = _operation, - SchemaId = _schemaId, - RecordKey = _recordKey - }; - if (_claimed) _session.Answered(_messageId, receipt); - _send(receipt); + if (_claimed) _session.Admitted(_messageId); + _send(AdmittedReceipt()); } /// The write was refused: remember the refusal (a resend repeats it) and send its one error.v1. @@ -240,7 +310,7 @@ namespace GameCult.Networking if (_answered) return; _answered = true; var refusal = CultNetErrorV1Message.From(error, _messageId); - if (_claimed) _session.Answered(_messageId, refusal); + if (_claimed) _session.Refused(_messageId, error); _send(refusal); } @@ -263,7 +333,15 @@ namespace GameCult.Networking _claimed = false; } - /// A missing id cannot be answered under rule 1; refuse it with no id. - public bool HasId => _messageId.Length > 0; + /// + /// Why the write's id cannot be used at all, before anything is remembered: it is missing (refused with no id) or longer + /// than UTF-8 bytes. Null when the id is usable. + /// + public CultNetErrorMessage? IdRefusal => + _messageId.Length == 0 + ? CultNetErrorMessage.ForMessageInvalid("messageId", "a write needs a messageId") + : System.Text.Encoding.UTF8.GetByteCount(_messageId) > CultNetWriteReceiptSession.MaxMessageIdBytes + ? CultNetErrorMessage.ForMessageInvalid("messageId", "messageId exceeds 256 bytes") + : null; } } diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs index e7900e7c..2bb59952 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -178,6 +178,46 @@ namespace GameCult.Networking.Tests Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); } + // The RUDP schema server implements no receipt rules, so it must not advertise them (write-receipt.md, Capability and the + // conformance table): whatever hello it sends lists neither frame. + [Test] + public async Task TheRudpSchemaServerNeverAdvertisesTheReceiptFrames() + { + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-rudp-no-receipts", + ConnectionId = ConnectionId + }); + var client = await Connect(server); + try + { + SendSchema(server, client, CultNetSchemaMessageSerialization.Serialize(new CultNetHelloMessage + { + RuntimeId = "receipt-client", + RuntimeKind = "test", + SupportedMessageVersions = [CultNetSchemaVersions.WriteAdmitted, CultNetSchemaVersions.ErrorV1] + })); + await server.PollAvailableAsync(16); + + var hellos = new List(); + foreach (var packet in Drain(client.Socket)) + foreach (var frame in client.Session.Receive(packet, 1).Delivered) + if (CultNetSchemaMessageSerialization.Deserialize(frame.Payload) is CultNetHelloMessage hello) + hellos.Add(hello); + + foreach (var hello in hellos) + { + var listed = hello.SupportedMessageVersions ?? Array.Empty(); + Assert.That(listed, Has.None.EqualTo(CultNetSchemaVersions.WriteAdmitted)); + Assert.That(listed, Has.None.EqualTo(CultNetSchemaVersions.ErrorV1)); + } + } + finally + { + client.Socket.Dispose(); + } + } + private static List ErrorsReceivedBy(Client client) { var errors = new List(); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index cab649e2..0cbc0c0b 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -873,5 +873,192 @@ namespace GameCult.Networking.Tests { Assert.That(new CultNetSchemaWriteForwarderOptions().ResultWindow, Is.EqualTo(CultNetWriteCompletion.ReceiptResultWindow)); } + + // ---- Heimdall round two ---- + + private static string IdOfBytes(int bytes) => new string('i', bytes); + + [Test] + public async Task AnIdOver256BytesIsRefusedBeforeAnythingIsRemembered() + { + var (rig, wire) = await Capable(); + var tooLong = IdOfBytes(257); + + await rig.Put(wire, tooLong, KeyA, "one"); + await rig.Put(wire, tooLong, KeyA, "one"); + + Assert.That(wire.Sent, Has.Count.EqualTo(2)); + foreach (var sent in wire.Sent) + { + var error = AssertRefused(sent, tooLong, "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("messageId")); + Assert.That(error.Error, Is.EqualTo("message_invalid: messageId exceeds 256 bytes")); + } + + Assert.That(rig.TextOf(KeyA), Is.Null); + // Nothing was remembered: the whole table is still free for admitted writes. + wire.Sent.Clear(); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + await rig.Put(wire, $"w{index}", new CultRecordKey($"tests:receipt:{index}"), "x"); + Assert.That(wire.Sent.OfType().Count(), Is.EqualTo(CultNetWriteReceiptSession.MaxRememberedWrites)); + } + + [Test] + public async Task TheIdLimitCountsUtf8BytesNotCharacters() + { + var (rig, wire) = await Capable(); + var atLimit = IdOfBytes(256); + var multibyte = new string('\u00e9', 129); // 129 chars, 258 bytes + + await rig.Put(wire, atLimit, KeyA, "one"); + await rig.Put(wire, multibyte, KeyB, "two"); + + AssertAdmitted(wire.Sent[0], atLimit, "put", KeyA.Value); + AssertRefused(wire.Sent[1], multibyte, "message_invalid"); + } + + [Test] + public void ARememberedEntryHoldsOnlyFixedSizeDigestsNotTheStrings() + { + var flags = BindingFlags.NonPublic | BindingFlags.Instance; + var table = typeof(CultNetWriteReceiptSession).GetFields(flags) + .Where(f => f.FieldType.IsGenericType && f.FieldType.GetGenericTypeDefinition() == typeof(Dictionary<,>)); + Assert.That(table, Is.Not.Empty); + foreach (var field in table) + { + var args = field.FieldType.GetGenericArguments(); + Assert.That(args[0].IsValueType, Is.True, $"{field.Name} is keyed by {args[0]}"); + foreach (var held in args[1].IsValueType || args[1] == typeof(string) ? new[] { args[1] } : args[1].GetFields(flags).Select(f => f.FieldType).Concat(args[1].GetProperties(flags | BindingFlags.Public).Select(p => p.PropertyType))) + { + Assert.That(held == typeof(string) || held == typeof(byte[]), Is.False, $"{field.Name} holds {held}"); + } + } + } + + [Test] + public async Task AnAdmittedResendIsAnsweredFromDigestsWithTheResendsOwnBody() + { + var (rig, wire) = await Capable(); + await rig.Put(wire, "m1", KeyA, "one"); + await rig.Put(wire, "m1", KeyA, "one"); + + AssertAdmitted(wire.Sent[1], "m1", "put", KeyA.Value); + Assert.That(rig.LogLength, Is.EqualTo(1)); + } + + [Test] + public async Task ARefusalIsReplayedForAResendAfterTheDigestChange() + { + var (rig, wire) = await Capable(); + rig.Authority.Denied.Add(KeyA.Value); + await rig.Put(wire, "m1", KeyA, "one"); + await rig.Put(wire, "m1", KeyA, "one"); + + AssertRefused(wire.Sent[0], "m1", "write_not_authorized"); + AssertRefused(wire.Sent[1], "m1", "write_not_authorized"); + Assert.That(rig.Authority.Asked, Is.EqualTo(1)); + } + + [Test] + public async Task APutWithNoDocumentIsRefusedNamingTheDocumentWithFixedText([Values] bool capable) + { + var rig = Build(); + var wire = new Wire(); + if (capable) await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + var message = rig.PutOf("m1", KeyA, "one"); + message.Document = null!; + + await rig.PutHandler(message, wire.Peer); + + var sent = wire.Sent.Single(); + var error = capable ? AssertRefused(sent, "m1", "message_invalid").ToV0() : (CultNetErrorMessage)sent; + Assert.That(error.Code, Is.EqualTo("message_invalid")); + Assert.That(error.Details!.Field, Is.EqualTo("document")); + Assert.That(error.Error, Is.EqualTo("message_invalid: document is required")); + } + + [Test] + public async Task APutWithNoSchemaIdIsRefusedNamingTheSchemaIdWithFixedText() + { + var (rig, wire) = await Capable(); + var message = rig.PutOf("m1", KeyA, "one"); + message.Document.SchemaId = null!; + + await rig.PutHandler(message, wire.Peer); + + var error = AssertRefused(wire.Sent.Single(), "m1", "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("schemaId")); + Assert.That(error.Error, Is.EqualTo("message_invalid: schemaId is required")); + } + + [Test] + public async Task ADeleteWithNoSchemaIdIsRefusedNamingTheSchemaIdWithFixedText([Values] bool capable) + { + var rig = Build(); + var wire = new Wire(); + if (capable) await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + var message = rig.DeleteOf("d1", KeyA); + message.SchemaId = null!; + + await rig.DeleteHandler(message, wire.Peer); + + var sent = wire.Sent.Single(); + var error = capable ? AssertRefused(sent, "d1", "message_invalid").ToV0() : (CultNetErrorMessage)sent; + Assert.That(error.Details!.Field, Is.EqualTo("schemaId")); + Assert.That(error.Error, Is.EqualTo("message_invalid: schemaId is required")); + } + + [Test] + public async Task ARefusalAtTheCapIsRememberedSoAResendReplaysItAndADifferentContentResendIsRefused() + { + var (rig, wire) = await Capable(); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + await rig.Put(wire, $"w{index}", new CultRecordKey($"tests:receipt:{index}"), "x"); + wire.Sent.Clear(); + + await rig.Put(wire, "over", KeyA, "x"); + await rig.Put(wire, "over", KeyA, "x"); + await rig.Put(wire, "over", KeyB, "other"); + + AssertRefused(wire.Sent[0], "over", "limit_reached"); + AssertRefused(wire.Sent[1], "over", "limit_reached"); + var changed = AssertRefused(wire.Sent[2], "over", "message_invalid"); + Assert.That(changed.Details!.Field, Is.EqualTo("messageId")); + Assert.That(rig.TextOf(KeyA), Is.Null); + } + + private static byte[] HashOf(string text) => CultNetWriteContentHash.Compute("put", "s", text, new byte[] { 1 }); + + [Test] + public void AnAbandonedClaimFreesNoSlotSoARefusedIdNeverBecomesAdmissible() + { + var session = new CultNetWriteReceiptSession(); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + Assert.That(session.TryBegin($"w{index}", HashOf($"k{index}"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.New)); + Assert.That(session.TryBegin("over", HashOf("o"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + + session.Abandoned("w0"); + + Assert.That(session.TryBegin("over", HashOf("o"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + Assert.That(session.TryBegin("another", HashOf("a"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + // The abandoned id itself is judged afresh, in its own slot. + Assert.That(session.TryBegin("w0", HashOf("k0"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.New)); + } + + [Test] + public void TheRefusalSetIsBoundedAndNeverAdmitsAnId() + { + var session = new CultNetWriteReceiptSession(); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + session.TryBegin($"w{index}", HashOf($"k{index}"), out _); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites + 5; index++) + Assert.That(session.TryBegin($"o{index}", HashOf("o"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + + session.Abandoned("w1"); + + Assert.That(session.TryBegin("o1030", HashOf("o"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + } } } From 1447ccd8dd6846cd06bef25d60644f978bcc22cd Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:09:28 +0000 Subject: [PATCH 041/117] cultnet-rs, cultmesh-rs: write_admitted in spec field order; server-wide remembered-write budget over digest entries; id size limit; limit refusals remembered; unreliable writes refused; local failure is its own goodbye Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 218 +++++++++++++++--- .../cultmesh-rs/tests/rudp_document_server.rs | 126 +++++++++- packages/cultnet-rs/src/contracts.rs | 18 +- packages/cultnet-rs/src/rudp.rs | 10 + packages/cultnet-rs/tests/write_receipt.rs | 21 +- 5 files changed, 359 insertions(+), 34 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index c6825c8e..40dfb767 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,7 +3,7 @@ use cultnet_rs::{ CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, is_permanent_send_error, readable_message_id, send_error_code, write_content_hash, + encode_rudp_packet, is_permanent_send_error, readable_message_id, message_id_digest, send_error_code, write_content_hash, }; use std::collections::{BTreeMap, HashMap}; #[cfg(test)] @@ -189,6 +189,15 @@ pub struct CultMeshRudpDocumentServerOptions { pub resend_delay: Duration, pub max_pending_reliable_packets_per_session: usize, pub max_fragment_bytes: usize, + /// Server-wide memory budget for the remembered writes of all receipt sessions + /// (write-receipt.md, "Remembered-write limits"): at most this many bytes, counted as + /// `REMEMBERED_ENTRY_BYTES` per entry. A new write over it is refused `limit_reached`, the session kept. + pub max_remembered_write_bytes: usize, +} + +impl CultMeshRudpDocumentServerOptions { + /// What one remembered write is charged: the digests of its id and content, and its answer. + pub const REMEMBERED_ENTRY_BYTES: usize = 32 + 32 + 1; } impl Default for CultMeshRudpDocumentServerOptions { @@ -204,6 +213,7 @@ impl Default for CultMeshRudpDocumentServerOptions { resend_delay: Duration::from_millis(50), max_pending_reliable_packets_per_session: 1024, max_fragment_bytes: 1200, + max_remembered_write_bytes: 32 * 1024 * 1024, } } } @@ -275,6 +285,10 @@ pub enum CultMeshRudpRejectionReason { /// A receipt-capable session already remembers `MAX_REMEMBERED_WRITES_PER_SESSION` writes and takes no new /// id. Nothing reached the sink. WriteLimitReached, + /// A receipt-capable session sent a message id over `MAX_MESSAGE_ID_BYTES`. Nothing was remembered. + MessageIdTooLong, + /// A receipt-capable session sent a write on an unreliable channel, where no answer could be owed once. + ChannelNotReliable, } impl std::fmt::Display for CultMeshRudpRejectionReason { @@ -285,6 +299,8 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { Self::MessageUnreadable(error) => write!(f, "message could not be read: {error}"), Self::MessageIdReused => f.write_str("the message id was already used for different content"), Self::WriteLimitReached => f.write_str("the session remembers as many writes as it will"), + Self::MessageIdTooLong => f.write_str("the message id is longer than 256 bytes"), + Self::ChannelNotReliable => f.write_str("a receipt session's write arrived on an unreliable channel"), Self::DocumentUnservable { response_bytes, max_snapshot_response_bytes, @@ -325,7 +341,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { /// The texts a refused peer is sent, one per rejection variant. They are fixed: /// a sink's or source's error text can quote what a peer sent, so it stays with /// the caller that receives the rejection and never reaches the wire. -pub(crate) const REFUSAL_TEXTS: [&str; 13] = [ +pub(crate) const REFUSAL_TEXTS: [&str; 15] = [ "the catalog refused the document", "the document can never be served", "the snapshot source failed", @@ -339,12 +355,40 @@ pub(crate) const REFUSAL_TEXTS: [&str; 13] = [ "the message could not be read", "the message id was already used for different content", "the session's remembered-write limit is full", + "messageId exceeds 256 bytes", + "receipt sessions require the reliable channel", ]; impl CultMeshRudpRejectionReason { /// The fixed text the refused peer is sent for this variant. fn refusal_text(&self) -> &'static str { - REFUSAL_TEXTS[match self { + REFUSAL_TEXTS[usize::from(self.refusal_index())] + } + + /// The reason a remembered refusal replays as. Its payload (a sink's text, a size) is not remembered: the + /// peer is only ever sent the variant's fixed text, and the caller's rejection is reported once, at first sight. + fn from_refusal_index(index: u8) -> Self { + match index { + 0 => Self::SinkRefused(String::new()), + 1 => Self::DocumentUnservable { response_bytes: 0, max_snapshot_response_bytes: 0, fragment_count: 0, max_fragment_count: 0 }, + 2 => Self::SnapshotSourceFailed(String::new()), + 3 => Self::SnapshotTooManyDocuments { documents: 0, max_snapshot_documents: 0 }, + 4 => Self::SnapshotResponseTooLarge { response_bytes: 0, max_snapshot_response_bytes: 0 }, + 5 => Self::ResponseEncodingFailed(String::new()), + 6 => Self::PayloadBudgetFull, + 7 => Self::ResponseQueueFailed(String::new()), + 8 => Self::ResponseSendFailed(""), + 9 => Self::DeleteNotSupported, + 10 => Self::MessageUnreadable(String::new()), + 11 => Self::MessageIdReused, + 12 => Self::WriteLimitReached, + 13 => Self::MessageIdTooLong, + _ => Self::ChannelNotReliable, + } + } + + fn refusal_index(&self) -> u8 { + match self { Self::SinkRefused(_) => 0, Self::DocumentUnservable { .. } => 1, Self::SnapshotSourceFailed(_) => 2, @@ -358,7 +402,9 @@ impl CultMeshRudpRejectionReason { Self::MessageUnreadable(_) => 10, Self::MessageIdReused => 11, Self::WriteLimitReached => 12, - }] + Self::MessageIdTooLong => 13, + Self::ChannelNotReliable => 14, + } } } @@ -437,6 +483,8 @@ impl CultMeshRudpApplicationRejection { (CultMeshRudpRejectionReason::MessageIdReused, _) => { message_invalid_message("messageId", text) } + (CultMeshRudpRejectionReason::MessageIdTooLong, _) => message_invalid_message("messageId", text), + (CultMeshRudpRejectionReason::ChannelNotReliable, _) => message_invalid_message("channel", text), (CultMeshRudpRejectionReason::WriteLimitReached, _) => CultNetMessage::Error { error: format!("limit_reached: {text}"), code: Some(CultNetErrorCode::LimitReached), @@ -475,6 +523,8 @@ enum SessionEnd { Rejected(CultNetMessage), /// The session owes its peer a packet that can never be sent as built. Unsendable(std::io::Error), + /// The server could not answer a write it owed an answer to. Not a refusal of anything the peer sent. + LocalFailure, } fn write_rejection( @@ -512,9 +562,31 @@ struct SessionEntry { /// Answered writes of a receipt-capable session: message id -> (content hash, the answer given), refusals /// included, so a resend of the same content is answered again without being decided again and the same /// id with other content is refused. - answered_writes: HashMap, + answered_writes: HashMap<[u8; 32], RememberedWrite>, + /// Writes refused `limit_reached`, kept so their resend replays the refusal. A bounded set outside the + /// entry budget (`MAX_REMEMBERED_WRITES_PER_SESSION`); past it a new write is refused unremembered. + limit_refusals: HashMap<[u8; 32], [u8; 32]>, } +/// What the server remembers of a write it answered: digests only, so an entry costs +/// `REMEMBERED_ENTRY_BYTES` whatever the id or payload size. The answer's body is rebuilt from the resend, +/// whose content hash has been checked equal to the original's. +#[derive(Clone, Copy)] +struct RememberedWrite { + content_hash: [u8; 32], + answer: StoredAnswer, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum StoredAnswer { + Admitted, + /// A refusal, by its `REFUSAL_TEXTS` index (`CultMeshRudpRejectionReason::refusal_index`). + Refused(u8), +} + +/// The longest `messageId` a receipt session may send, in UTF-8 bytes. +const MAX_MESSAGE_ID_BYTES: usize = 256; + /// The most answered writes a receipt-capable session remembers; a session at the limit refuses new writes /// (`WriteLimitReached`) rather than forget an id it promised to answer idempotently. const WRITE_ADMITTED_VERSION: &str = "cultnet.write_admitted.v0"; @@ -663,7 +735,9 @@ where && packet.channel_id == "schema" && !packet.reliable { - return Ok(CultMeshRudpPollOutcome::Handled); + // A receipt session's write that could be lost without trace cannot be answered once: it is + // refused on the reliable channel, never applied. Anything else unreliable is dropped as before. + return self.refuse_unreliable_write(key, &packet.payload); } let payload_charge = if packet.packet_type == CultNetRudpPacketType::Data { packet.payload.len() @@ -789,7 +863,7 @@ where Answered::Sent => {} Answered::Unsendable(error) => unsendable = Some(error), Answered::Unqueueable => { - self.end_session(key, SessionEnd::Refused)?; + self.end_session(key, SessionEnd::LocalFailure)?; return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } } @@ -814,7 +888,7 @@ where // An unanswerable write is answered by ending the session: its peer settles it // as `session_ended`, never as silence. Answered::Unqueueable => { - self.end_session(key, SessionEnd::Refused)?; + self.end_session(key, SessionEnd::LocalFailure)?; return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } } @@ -897,6 +971,39 @@ where }) } + /// Refuses a put or delete that arrived on an unreliable channel of a receipt session with + /// `message_invalid` (`details.field` `channel`) naming its id. Nothing is remembered or applied: the id + /// stays free for the same write sent reliably. + fn refuse_unreliable_write( + &mut self, + key: CultMeshRudpSessionKey, + payload: &[u8], + ) -> Result { + if !self.receipts(key) { + return Ok(CultMeshRudpPollOutcome::Handled); + } + let Ok(message) = decode_cultnet_message_from_slice(payload, CultNetWireContract::CultNetSchemaV0) else { + return Ok(CultMeshRudpPollOutcome::Handled); + }; + let (operation, message_id, record_key) = match message { + CultNetMessage::DocumentPutRaw { message_id, document } => { + (CultMeshRudpApplicationOperation::DocumentPutRaw, message_id, document.record_key) + } + CultNetMessage::DocumentDelete { message_id, record_key, .. } => { + (CultMeshRudpApplicationOperation::DocumentDelete, message_id, record_key) + } + _ => return Ok(CultMeshRudpPollOutcome::Handled), + }; + let rejection = write_rejection(key, operation, &message_id, &record_key, CultMeshRudpRejectionReason::ChannelNotReliable); + let refusal = rejection.refusal_message(true); + match self.send_on_live_session(key, &refusal, true)? { + Answered::Sent => {} + Answered::Unsendable(error) => self.end_session(key, SessionEnd::Unsendable(error))?, + Answered::Unqueueable => self.end_session(key, SessionEnd::LocalFailure)?, + } + Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)) + } + /// Expire idle sessions and resend reliable packets whose deadline passed. pub fn maintain(&mut self) -> Result { let now = self.clock.now_monotonic_millis(); @@ -1011,6 +1118,7 @@ where receipts: false, capability_settled: false, answered_writes: HashMap::new(), + limit_refusals: HashMap::new(), }, ); accept @@ -1067,6 +1175,9 @@ where schema_id: receipt.document.schema_id.clone(), record_key: receipt.document.record_key.clone(), }; + if message_id.len() > MAX_MESSAGE_ID_BYTES { + return Ok(Some(rejection(CultMeshRudpRejectionReason::MessageIdTooLong))); + } let content_hash = match write_content_hash( "put", &receipt.document.schema_id, @@ -1082,20 +1193,21 @@ where }; // The id is looked up first: a resend is answered with what the session already said, and // the sink is asked only about an id the session has not seen. + let id_digest = message_id_digest(&message_id); if let ControlFlow::Break(outcome) = - self.seen_write(key, now, &message_id, content_hash, &rejection)? + self.seen_write(key, now, id_digest, content_hash, &admitted, &rejection)? { return Ok(outcome); } match self.admit_put(receipt, true) { Ok(()) => { // Admitted: the sink holds it. Only now is a receipt owed, never before the commit point. - self.remember(key, &message_id, content_hash, admitted.clone()); + self.remember(key, id_digest, content_hash, StoredAnswer::Admitted); return self.answer_sent(key, now, &admitted, &rejection); } Err(reason) => { let refused = rejection(reason); - self.remember(key, &message_id, content_hash, refused.refusal_message(true)); + self.remember(key, id_digest, content_hash, StoredAnswer::Refused(refused.reason.refusal_index())); return Ok(Some(refused)); } } @@ -1278,6 +1390,9 @@ where if !self.receipts(key) { return Ok(Some(rejection(CultMeshRudpRejectionReason::DeleteNotSupported))); } + if message_id.len() > MAX_MESSAGE_ID_BYTES { + return Ok(Some(rejection(CultMeshRudpRejectionReason::MessageIdTooLong))); + } let content_hash = match write_content_hash("delete", &schema_id, &record_key, None) { Ok(hash) => hash, Err(error) => { @@ -1286,13 +1401,21 @@ where )))); } }; + let id_digest = message_id_digest(&message_id); + // A delete is never admitted; the body is the receipt it would be answered with. + let would_admit = CultNetMessage::WriteAdmitted { + message_id: message_id.clone(), + operation: "delete".into(), + schema_id: schema_id.clone(), + record_key: record_key.clone(), + }; if let ControlFlow::Break(outcome) = - self.seen_write(key, now, &message_id, content_hash, &rejection)? + self.seen_write(key, now, id_digest, content_hash, &would_admit, &rejection)? { return Ok(outcome); } let refused = rejection(CultMeshRudpRejectionReason::DeleteNotSupported); - self.remember(key, &message_id, content_hash, refused.refusal_message(true)); + self.remember(key, id_digest, content_hash, StoredAnswer::Refused(refused.reason.refusal_index())); return Ok(Some(refused)); } _ => {} @@ -1300,36 +1423,63 @@ where Ok(None) } - /// What a receipt-capable session already knows of `message_id`. `Continue` is a first sighting that - /// the session has room to remember. A repeat with the same content is answered with the stored answer, - /// a repeat with other content (a put or a delete of any record) is `MessageIdReused`, and a session - /// that remembers its limit takes no new id. + /// What a receipt-capable session already knows of a write, by the digest of its id. `Continue` is a first + /// sighting that fits both bounds. A repeat with the same content is answered with its remembered answer + /// (rebuilt from this resend, whose content hash is the original's), a repeat with other content is + /// `MessageIdReused`. A first sighting past the session table or the server-wide budget is refused + /// `limit_reached` for that write alone, and that refusal is remembered (`limit_refusals`) so its resend + /// replays it; no slot is freed, so the id is never admitted later in the session. fn seen_write( &mut self, key: CultMeshRudpSessionKey, now: u64, - message_id: &str, + id_digest: [u8; 32], content_hash: [u8; 32], + admitted: &CultNetMessage, rejection: &impl Fn(CultMeshRudpRejectionReason) -> CultMeshRudpApplicationRejection, ) -> Result>> { - let (prior, remembered) = self - .sessions - .get(&key) - .map(|entry| (entry.answered_writes.get(message_id).cloned(), entry.answered_writes.len())) - .unwrap_or_default(); + let Some(entry) = self.sessions.get(&key) else { + return Ok(ControlFlow::Continue(())); + }; + let prior = entry.answered_writes.get(&id_digest).copied().or_else(|| { + entry.limit_refusals.get(&id_digest).map(|&content_hash| RememberedWrite { + content_hash, + answer: StoredAnswer::Refused(CultMeshRudpRejectionReason::WriteLimitReached.refusal_index()), + }) + }); + let session_full = entry.answered_writes.len() >= MAX_REMEMBERED_WRITES_PER_SESSION; + let refusal_room = entry.limit_refusals.len() < MAX_REMEMBERED_WRITES_PER_SESSION; + let remembered_total: usize = self.sessions.values().map(|entry| entry.answered_writes.len()).sum(); + let budget_full = remembered_total + .checked_add(1) + .and_then(|entries| entries.checked_mul(CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES)) + .is_none_or(|bytes| bytes > self.options.max_remembered_write_bytes); Ok(ControlFlow::Break(match prior { - Some((hash, answer)) if hash == content_hash => self.answer_sent(key, now, &answer, rejection)?, + Some(remembered) if remembered.content_hash == content_hash => { + let answer = match remembered.answer { + StoredAnswer::Admitted => admitted.clone(), + StoredAnswer::Refused(index) => { + rejection(CultMeshRudpRejectionReason::from_refusal_index(index)).refusal_message(true) + } + }; + self.answer_sent(key, now, &answer, rejection)? + } Some(_) => Some(rejection(CultMeshRudpRejectionReason::MessageIdReused)), - None if remembered >= MAX_REMEMBERED_WRITES_PER_SESSION => { + None if session_full || budget_full => { + if refusal_room { + if let Some(entry) = self.sessions.get_mut(&key) { + entry.limit_refusals.insert(id_digest, content_hash); + } + } Some(rejection(CultMeshRudpRejectionReason::WriteLimitReached)) } None => return Ok(ControlFlow::Continue(())), })) } - fn remember(&mut self, key: CultMeshRudpSessionKey, message_id: &str, content_hash: [u8; 32], answer: CultNetMessage) { + fn remember(&mut self, key: CultMeshRudpSessionKey, id_digest: [u8; 32], content_hash: [u8; 32], answer: StoredAnswer) { if let Some(entry) = self.sessions.get_mut(&key) { - entry.answered_writes.insert(message_id.to_owned(), (content_hash, answer)); + entry.answered_writes.insert(id_digest, RememberedWrite { content_hash, answer }); } } @@ -1346,7 +1496,7 @@ where if self.send_reliable_schema(key, now, answer) { return Ok(None); } - self.end_session(key, SessionEnd::Refused)?; + self.end_session(key, SessionEnd::LocalFailure)?; Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseSendFailed( "the write answer could not be sent".into(), )))) @@ -1463,10 +1613,12 @@ where let Some(mut entry) = self.sessions.remove(&key) else { return Ok(()); }; + let local_failure = matches!(end, SessionEnd::LocalFailure); let (refusal, mut unsendable) = match end { SessionEnd::Refused => (Vec::new(), None), SessionEnd::Rejected(refusal) => (self.refusal_packets(&mut entry.session, &refusal, false), None), SessionEnd::Unsendable(error) => (Vec::new(), Some(error)), + SessionEnd::LocalFailure => (Vec::new(), None), }; entry.session.reset_peer_state(); let nothing = entry.session.create_ack(); @@ -1478,9 +1630,10 @@ where break; } } - let goodbye = match &unsendable { - Some(error) => entry.session.end_unsendable_session(error), - None => entry.session.end_refused_session(), + let goodbye = match (&unsendable, local_failure) { + (Some(error), _) => entry.session.end_unsendable_session(error), + (None, true) => entry.session.end_local_failure_session(), + (None, false) => entry.session.end_refused_session(), }; // The goodbye is the session's last datagram. If it too can never be // sent, nobody is left to tell. @@ -1603,6 +1756,9 @@ fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.session_max_lifetime.is_zero() { return Err(anyhow!("session_max_lifetime must be greater than zero")); } + if options.max_remembered_write_bytes == 0 { + return Err(anyhow!("max_remembered_write_bytes must be greater than zero")); + } if options.max_admitted_payload_bytes == 0 { return Err(anyhow!( "max_admitted_payload_bytes must be greater than zero" diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 79e9e14c..0957d008 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2332,8 +2332,9 @@ fn a_full_dedupe_table_refuses_new_writes_and_keeps_the_session_and_its_receipts send_reliable(&mut peer, &put("w0", "note:0", vec![1]))?; let answers = frames_after(&mut server, &mut peer, 2)?; assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); - let CultNetMessage::ErrorV1 { error, .. } = &answers[0] else { unreachable!() }; + let CultNetMessage::ErrorV1 { error, details, .. } = &answers[0] else { unreachable!() }; assert_eq!(error, "limit_reached: the session's remembered-write limit is full", "the cap has its own fixed text"); + assert_eq!(details, &Some(CultNetErrorDetails::default()), "limit_reached names no field: details all null"); assert_eq!(answers[1], admitted("w0", "note:0"), "a remembered id is still answered idempotently"); assert_eq!(server.session_count(), 1); Ok(()) @@ -2423,6 +2424,17 @@ fn a_receipt_that_cannot_be_queued_ends_the_session_instead_of_leaving_the_write } } assert_eq!(server.session_count(), 0, "a receipt the queue could not take ended the session"); + // The goodbye names a local failure: it is not the "session refused a packet" goodbye of a protocol refusal. + let mut reason = None; + for _ in 0..200 { + while peer.receive_once()?.is_some() {} + if let Some(found) = peer.disconnect_reason() { + reason = Some(found.to_vec()); + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(reason.expect("the peer was told the session ended"), b"server could not answer a write".to_vec()); Ok(()) } @@ -2623,3 +2635,115 @@ fn a_put_that_reuses_a_deletes_id_is_message_invalid() -> Result<()> { assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "the reusing put was not applied"); Ok(()) } + +fn error_text_and_field(message: &CultNetMessage) -> (String, Option) { + let CultNetMessage::ErrorV1 { error, details, .. } = message else { panic!("expected error.v1, got {message:?}") }; + (error.clone(), details.as_ref().and_then(|d| d.field.clone())) +} + +/// The server-wide budget counts fixed-size entries across sessions; a write over it is refused for that write +/// alone, the refusal is remembered (a resend replays it, no slot is freed), and every session is kept. +#[test] +fn a_server_wide_memory_budget_refuses_the_write_over_it_and_keeps_the_sessions() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 2 * CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES, + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let mut a = client(server.local_addr()?, 230)?; + let mut b = client(server.local_addr()?, 231)?; + connect(&mut server, &mut [&mut a, &mut b])?; + send_reliable(&mut a, &receipt_hello())?; + send_reliable(&mut b, &receipt_hello())?; + send_reliable(&mut a, &put("a1", "note:a1", vec![1]))?; + send_reliable(&mut b, &put("b1", "note:b1", vec![1]))?; + for _ in 0..200 { + server.poll_once()?; + a.poll_resends()?; + b.poll_resends()?; + while a.receive_once()?.is_some() {} + while b.receive_once()?.is_some() {} + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); + send_reliable(&mut a, &put("a2", "note:a2", vec![1]))?; + send_reliable(&mut a, &put("a2", "note:a2", vec![1]))?; + let answers = frames_after(&mut server, &mut a, 2)?; + for answer in &answers { + assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::LimitReached), Some("a2".into())), "the resend replays the refusal"); + let CultNetMessage::ErrorV1 { details, .. } = answer else { unreachable!() }; + assert_eq!(details, &Some(CultNetErrorDetails::default())); + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2, "the refused write was not applied"); + assert_eq!(server.session_count(), 2, "both sessions are kept"); + send_reliable(&mut a, &put("a1", "note:a1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut a, 1)?, vec![admitted("a1", "note:a1")], "a remembered id is still answered"); + Ok(()) +} + +/// A message id over 256 bytes is refused before anything is remembered: no entry, no budget charge. +#[test] +fn a_message_id_over_256_bytes_is_refused_before_anything_is_remembered() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES, + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 232)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + let long = "x".repeat(257); + send_reliable(&mut peer, &put(&long, "note:1", vec![1]))?; + send_reliable(&mut peer, &put(&long, "note:1", vec![1]))?; + send_reliable(&mut peer, &put(&"y".repeat(256), "note:2", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 3)?; + for answer in &answers[..2] { + assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::MessageInvalid), Some(long.clone()))); + assert_eq!(error_text_and_field(answer), ("message_invalid: messageId exceeds 256 bytes".into(), Some("messageId".into()))); + } + assert_eq!(answers[2], admitted(&"y".repeat(256), "note:2"), "256 bytes is allowed, and the long id charged no budget"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "the long-id writes were never applied"); + Ok(()) +} + +/// A write on an unreliable channel of a receipt session is refused, not dropped and not accepted. +#[test] +fn an_unreliable_write_on_a_receipt_session_is_refused_message_invalid_naming_the_channel() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 233)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let payload = encode_cultnet_message_to_vec(&put("u1", "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + let mut answers = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + let refusals: Vec<_> = answers.iter().filter(|answer| matches!(answer, CultNetMessage::ErrorV1 { .. })).collect(); + assert_eq!(refusals.len(), 1, "the unreliable write is answered, not dropped: {answers:?}"); + assert_eq!(refusal_of(refusals[0]), (Some(CultNetErrorCode::MessageInvalid), Some("u1".into()))); + assert_eq!( + error_text_and_field(refusals[0]), + ("message_invalid: receipt sessions require the reliable channel".into(), Some("channel".into())) + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "nothing was applied"); + assert_eq!(server.session_count(), 1); + // The id was not consumed: the same write on the reliable channel is admitted. + peer.send(&put("u1", "note:1", vec![1]))?; + server.poll_once()?; + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + Ok(()) +} diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 1aa39d11..531fe7ec 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -569,7 +569,8 @@ pub fn encode_cultnet_message_for_wire( | CultNetMessage::ContentChunkRequest { .. } | CultNetMessage::ContentChunkResponse { .. } | CultNetMessage::Error { .. } - | CultNetMessage::ErrorV1 { .. } => encode_raw_cultnet_schema_message(message), + | CultNetMessage::ErrorV1 { .. } + | CultNetMessage::WriteAdmitted { .. } => encode_raw_cultnet_schema_message(message), _ => Ok(cultcache_rs::decode_payload(&rmp_serde::to_vec( &serde_json::to_value(message)?, )?)?), @@ -1146,6 +1147,14 @@ fn encode_raw_cultnet_schema_message(message: &CultNetMessage) -> Result vec![ + (rmpv::Value::from("schemaVersion"), rmpv::Value::from("cultnet.write_admitted.v0")), + (rmpv::Value::from("messageId"), rmpv::Value::from(message_id.as_str())), + (rmpv::Value::from("operation"), rmpv::Value::from(operation.as_str())), + (rmpv::Value::from("schemaId"), rmpv::Value::from(schema_id.as_str())), + (rmpv::Value::from("recordKey"), rmpv::Value::from(record_key.as_str())), + ], CultNetMessage::DocumentPutRaw { message_id, document, @@ -1991,6 +2000,13 @@ pub fn write_content_hash( Ok(Sha256::digest(&bytes).into()) } +/// SHA-256 of a write's `messageId`: what a receipt server remembers in place of the id itself, so an +/// entry's memory does not depend on the id's length (write-receipt.md, "What is remembered"). +pub fn message_id_digest(message_id: &str) -> [u8; 32] { + use sha2::{Digest, Sha256}; + Sha256::digest(message_id.as_bytes()).into() +} + /// The `messageId` of a schema frame that does not decode as a message (write-receipt.md, rule 3: a refusal /// carries the write's id whenever the frame is readable enough to find it). `None` when the bytes are not /// a MessagePack map or hold no non-empty string `messageId`. diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 6a2aa20b..eae7a2ee 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -468,6 +468,14 @@ impl CultNetRudpSession { self.create_disconnect(RUDP_REFUSED_PACKET_REASON.to_vec()) } + /// Ends a session because the server itself could not answer a write it owes an answer to (its answer + /// could not be encoded, queued or sent). The peer did nothing wrong: this goodbye is not the + /// "session refused a packet" of a protocol refusal. + pub fn end_local_failure_session(&mut self) -> CultNetRudpPacket { + self.reset_peer_state(); + self.create_disconnect(RUDP_LOCAL_FAILURE_REASON.to_vec()) + } + /// Ends a session that owes its peer a packet that can never be sent as /// built (see `is_permanent_send_error`). Resending it would fail forever, /// and dropping it would leave a reliable sequence the peer waits on for @@ -2609,6 +2617,8 @@ where /// Sent to a peer whose session refused one of its packets, and reported to /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; +/// Goodbye reason of a session whose server could not answer a write (`end_local_failure_session`). +const RUDP_LOCAL_FAILURE_REASON: &[u8] = b"server could not answer a write"; const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; /// Prefix of the goodbye reason for a session that owed a packet which can /// never be sent; the failure's fixed name (`send_error_code`) follows it. diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 4ad7a826..6d06a68c 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -101,7 +101,7 @@ fn error_v1_limit_reached_is_the_spec_vector() { message_id: Some("put-1025".into()), }; let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap(); - assert_eq!(hex(&bytes), vector); + assert_eq!(bytes, unhex(vector), "encoded bytes must equal the spec vector"); let CultNetMessage::ErrorV1 { error, code, details, message_id } = decode_cultnet_message_from_slice(&unhex(vector), CultNetWireContract::CultNetSchemaV0).unwrap() else { @@ -113,6 +113,25 @@ fn error_v1_limit_reached_is_the_spec_vector() { assert_eq!(details, Some(CultNetErrorDetails::default())); } +/// Spec vectors `write_admitted_put` and `write_admitted_delete`: the encoder emits the spec's field order. +#[test] +fn write_admitted_encodes_to_the_spec_vector_bytes() { + let cases = [ + ("put-1", "put", "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"), + ("del-1", "delete", "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A564656C2D31A96F7065726174696F6EA664656C657465A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"), + ]; + for (id, operation, vector) in cases { + let message = CultNetMessage::WriteAdmitted { + message_id: id.into(), + operation: operation.into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + }; + let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!(bytes, unhex(vector)); + } +} + /// `limit_reached` is in the error.v1 enum only: a v0 error neither encodes nor decodes it. #[test] fn error_v0_refuses_the_v1_only_limit_reached_code() { From d654cb2b9980d712524f491ee4ddd1e28f8c9849 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:12:31 +0000 Subject: [PATCH 042/117] C#: tests kill the refusal-set bound and abandoned-rehash mutants Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetWriteReceiptServerTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 0cbc0c0b..eb20b345 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1060,5 +1060,32 @@ namespace GameCult.Networking.Tests Assert.That(session.TryBegin("o1030", HashOf("o"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); } + + [Test] + public void TheRefusalSetStopsRememberingAtItsBound() + { + var session = new CultNetWriteReceiptSession(); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedWrites; index++) + session.TryBegin($"w{index}", HashOf($"k{index}"), out _); + for (var index = 0; index < CultNetWriteReceiptSession.MaxRememberedLimitRefusals; index++) + session.TryBegin($"o{index}", HashOf("o"), out _); + + // Past the bound a refusal is not remembered: its changed-content resend is simply refused at the limit again. + Assert.That(session.TryBegin("late", HashOf("a"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + Assert.That(session.TryBegin("late", HashOf("b"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Full)); + // Inside the bound it is remembered. + Assert.That(session.TryBegin("o0", HashOf("changed"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.DifferentContent)); + } + + [Test] + public void AnAbandonedIdMayBeResentWithOtherContentBecauseItWasNeverSent() + { + var session = new CultNetWriteReceiptSession(); + session.TryBegin("m1", HashOf("a"), out _); + session.Abandoned("m1"); + + Assert.That(session.TryBegin("m1", HashOf("b"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.New)); + Assert.That(session.TryBegin("m1", HashOf("b"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Unanswered)); + } } } From ef5cd881119d6a16d5338c9cce09bf654ad46ed7 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:13:58 +0000 Subject: [PATCH 043/117] C#: v0 errors for snapshot, shard log and variant refusals carry fixed text, never an exception's message Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetDatabaseServer.cs | 7 ++-- .../CultNetDocumentRegistryVariantTests.cs | 1 + .../CultNetWriteReceiptServerTests.cs | 32 +++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 32688434..16967d49 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -266,7 +266,7 @@ namespace GameCult.Networking catch (Exception ex) { _server.Logger.LogError($"CultNet snapshot v1 request failed: {ex.Message}"); - peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); + peer.SendCultNet(new CultNetErrorMessage { Error = "the snapshot request could not be answered" }); } return Task.CompletedTask; @@ -286,7 +286,8 @@ namespace GameCult.Networking } catch (Exception ex) { - peer.SendCultNet(new CultNetErrorMessage { Error = ex.Message }); + _server.Logger.LogError($"CultNet shard log request failed: {ex.Message}"); + peer.SendCultNet(new CultNetErrorMessage { Error = "the shard log request could not be answered" }); } return Task.CompletedTask; @@ -754,7 +755,7 @@ namespace GameCult.Networking var refusedKey = change.GetType().GetProperty("Key")?.GetValue(change) is CultRecordKey changed ? changed.Value : string.Empty; lock (refused) refused.Add(refusedKey); - sendError(new CultNetErrorMessage { Error = refusal.Message }); + sendError(new CultNetErrorMessage { Error = "the record is a variant CultNet cannot carry" }); return null; } diff --git a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs index 64de1093..5713b0da 100644 --- a/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetDocumentRegistryVariantTests.cs @@ -418,6 +418,7 @@ namespace GameCult.Networking.Tests CultNetDatabaseChangeKind.Added, VariantKey, schemaId, database.Shards[0], cache.Get(VariantKey)!, previousDocument: null); Assert.That(Deliver(databaseServer, added, selection, refused, errors), Is.Null); Assert.That(errors, Has.Count.EqualTo(1)); + Assert.That(errors[0].Error, Is.EqualTo("the record is a variant CultNet cannot carry")); var removed = new CultNetDatabaseChange( CultNetDatabaseChangeKind.Removed, VariantKey, schemaId, database.Shards[0], document: null, previousDocument: cache.Get(VariantKey)!); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index eb20b345..09188c5f 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1087,5 +1087,37 @@ namespace GameCult.Networking.Tests Assert.That(session.TryBegin("m1", HashOf("b"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.New)); Assert.That(session.TryBegin("m1", HashOf("b"), out _), Is.EqualTo(CultNetWriteReceiptSession.Begin.Unanswered)); } + + // ---- Fixed text in v0 errors: an exception's own words never reach the wire ---- + + private static Func HandlerOf(Rig rig, string field) => + (Func)typeof(CultNetDatabaseServer) + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(rig.DatabaseServer)!; + + [Test] + public async Task ASnapshotFaultIsAnsweredWithFixedTextNotTheExceptionMessage() + { + var rig = Build(); + var wire = new Wire(); + + await HandlerOf(rig, "_snapshotHandlerV1")( + new CultNetSnapshotRequestV1Message { MessageId = "s1", Selection = null! }, wire.Peer); + + var error = (CultNetErrorMessage)wire.Sent.Single(); + Assert.That(error.Error, Is.EqualTo("the snapshot request could not be answered")); + } + + [Test] + public async Task AShardLogFaultIsAnsweredWithFixedTextNotTheExceptionMessage() + { + var rig = Build(); + var wire = new Wire(); + + await HandlerOf(rig, "_shardLogHandler")( + new CultNetShardLogRequestMessage { MessageId = "l1", ShardId = string.Empty }, wire.Peer); + + var error = (CultNetErrorMessage)wire.Sent.Single(); + Assert.That(error.Error, Is.EqualTo("the shard log request could not be answered")); + } } } From d7b36416c6b2530dea3277771795f67f99d6ef0a Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:30:36 +0000 Subject: [PATCH 044/117] cultmesh-rs: pin limit-refusal memory bound and legacy unreliable drop (mutant-driven tests) Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 0957d008..5dcee048 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2747,3 +2747,62 @@ fn an_unreliable_write_on_a_receipt_session_is_refused_message_invalid_naming_th assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); Ok(()) } + +/// A limit refusal is remembered like any refusal: the same id with other content is a reused id, not a fresh +/// limit refusal; and the remembered refusals are a bounded set (1024), past which a refusal is not remembered. +#[test] +fn limit_refusals_are_remembered_in_a_bounded_set() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 234)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("full", "note:full", vec![1]))?; + for index in 0..1024 { + send_reliable(&mut peer, &put(&format!("r{index}"), "note:r", vec![1]))?; + if index % 16 == 15 { + for _ in 0..40 { + server.poll_once()?; + peer.poll_resends()?; + while peer.receive_once()?.is_some() {} + } + } + } + // r0 was remembered as refused: other content under its id is a reused id. + send_reliable(&mut peer, &put("r0", "note:other", vec![2]))?; + let answers = frames_after(&mut server, &mut peer, 1)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("r0".into()))); + // The 1025th refusal found the set full: it was refused but not remembered. + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + send_reliable(&mut peer, &put("over", "note:over", vec![2]))?; + let answers = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::LimitReached), Some("over".into())), "not remembered: refused afresh"); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// A session that never declared the capability gets no refusal for an unreliable write: it is dropped as before. +#[test] +fn an_unreliable_write_on_a_legacy_session_is_dropped_silently() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 235)?; + let payload = encode_cultnet_message_to_vec(&put("u1", "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + assert!(peer.drain().is_empty(), "a legacy session is sent nothing"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + Ok(()) +} From c8f869e54e558d3d189446fa57233d5d2d4312d5 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:32:03 +0000 Subject: [PATCH 045/117] cultnet-rs, cultmesh-rs: null or missing document/schemaId refused naming the field; budget tests use literal bytes; unreliable-refusal local failure test Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 47 ++++++++++--- .../cultmesh-rs/tests/rudp_document_server.rs | 68 ++++++++++++++++++- packages/cultnet-rs/src/contracts.rs | 26 +++++++ 3 files changed, 128 insertions(+), 13 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 40dfb767..2b320578 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,7 +3,7 @@ use cultnet_rs::{ CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, is_permanent_send_error, readable_message_id, message_id_digest, send_error_code, write_content_hash, + encode_rudp_packet, is_permanent_send_error, readable_message_id, message_id_digest, missing_required_write_field, send_error_code, write_content_hash, }; use std::collections::{BTreeMap, HashMap}; #[cfg(test)] @@ -289,6 +289,10 @@ pub enum CultMeshRudpRejectionReason { MessageIdTooLong, /// A receipt-capable session sent a write on an unreliable channel, where no answer could be owed once. ChannelNotReliable, + /// A receipt-capable session's put or delete has a null or missing `document`. + DocumentMissing, + /// A receipt-capable session's put or delete has a null or missing `schemaId`. + SchemaIdMissing, } impl std::fmt::Display for CultMeshRudpRejectionReason { @@ -301,6 +305,8 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { Self::WriteLimitReached => f.write_str("the session remembers as many writes as it will"), Self::MessageIdTooLong => f.write_str("the message id is longer than 256 bytes"), Self::ChannelNotReliable => f.write_str("a receipt session's write arrived on an unreliable channel"), + Self::DocumentMissing => f.write_str("the write has no document"), + Self::SchemaIdMissing => f.write_str("the write has no schemaId"), Self::DocumentUnservable { response_bytes, max_snapshot_response_bytes, @@ -341,7 +347,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { /// The texts a refused peer is sent, one per rejection variant. They are fixed: /// a sink's or source's error text can quote what a peer sent, so it stays with /// the caller that receives the rejection and never reaches the wire. -pub(crate) const REFUSAL_TEXTS: [&str; 15] = [ +pub(crate) const REFUSAL_TEXTS: [&str; 17] = [ "the catalog refused the document", "the document can never be served", "the snapshot source failed", @@ -357,6 +363,8 @@ pub(crate) const REFUSAL_TEXTS: [&str; 15] = [ "the session's remembered-write limit is full", "messageId exceeds 256 bytes", "receipt sessions require the reliable channel", + "document is required", + "schemaId is required", ]; impl CultMeshRudpRejectionReason { @@ -383,7 +391,9 @@ impl CultMeshRudpRejectionReason { 11 => Self::MessageIdReused, 12 => Self::WriteLimitReached, 13 => Self::MessageIdTooLong, - _ => Self::ChannelNotReliable, + 14 => Self::ChannelNotReliable, + 15 => Self::DocumentMissing, + _ => Self::SchemaIdMissing, } } @@ -404,6 +414,8 @@ impl CultMeshRudpRejectionReason { Self::WriteLimitReached => 12, Self::MessageIdTooLong => 13, Self::ChannelNotReliable => 14, + Self::DocumentMissing => 15, + Self::SchemaIdMissing => 16, } } } @@ -484,6 +496,8 @@ impl CultMeshRudpApplicationRejection { message_invalid_message("messageId", text) } (CultMeshRudpRejectionReason::MessageIdTooLong, _) => message_invalid_message("messageId", text), + (CultMeshRudpRejectionReason::DocumentMissing, _) => message_invalid_message("document", text), + (CultMeshRudpRejectionReason::SchemaIdMissing, _) => message_invalid_message("schemaId", text), (CultMeshRudpRejectionReason::ChannelNotReliable, _) => message_invalid_message("channel", text), (CultMeshRudpRejectionReason::WriteLimitReached, _) => CultNetMessage::Error { error: format!("limit_reached: {text}"), @@ -844,13 +858,26 @@ where now_unix, now_monotonic, )?, - Err(error) => Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::Unreadable, - message_id: readable_message_id(&frame.payload).unwrap_or_default(), - record_key: None, - reason: CultMeshRudpRejectionReason::MessageUnreadable(format!("{error:#}")), - }), + Err(error) => { + // A receipt session's put or delete missing a required field is a refused write that + // names the field, not an unreadable frame. + let missing = if receipt_session { missing_required_write_field(&frame.payload) } else { None }; + Some(CultMeshRudpApplicationRejection { + session: key, + operation: match missing { + Some(("put", _)) => CultMeshRudpApplicationOperation::DocumentPutRaw, + Some(_) => CultMeshRudpApplicationOperation::DocumentDelete, + None => CultMeshRudpApplicationOperation::Unreadable, + }, + message_id: readable_message_id(&frame.payload).unwrap_or_default(), + record_key: None, + reason: match missing { + Some((_, "document")) => CultMeshRudpRejectionReason::DocumentMissing, + Some(_) => CultMeshRudpRejectionReason::SchemaIdMissing, + None => CultMeshRudpRejectionReason::MessageUnreadable(format!("{error:#}")), + }, + }) + } }; if let Some(rejection) = rejection { // A refused delete or unreadable frame below the cap is answered on the live session, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 5dcee048..9a8c4ed3 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2646,7 +2646,7 @@ fn error_text_and_field(message: &CultNetMessage) -> (String, Option) { #[test] fn a_server_wide_memory_budget_refuses_the_write_over_it_and_keeps_the_sessions() -> Result<()> { let options = CultMeshRudpDocumentServerOptions { - max_remembered_write_bytes: 2 * CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES, + max_remembered_write_bytes: 2 * 65, ..Default::default() }; let sink = Sink::default(); @@ -2685,7 +2685,7 @@ fn a_server_wide_memory_budget_refuses_the_write_over_it_and_keeps_the_sessions( #[test] fn a_message_id_over_256_bytes_is_refused_before_anything_is_remembered() -> Result<()> { let options = CultMeshRudpDocumentServerOptions { - max_remembered_write_bytes: CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES, + max_remembered_write_bytes: 65, ..Default::default() }; let sink = Sink::default(); @@ -2753,7 +2753,7 @@ fn an_unreliable_write_on_a_receipt_session_is_refused_message_invalid_naming_th #[test] fn limit_refusals_are_remembered_in_a_bounded_set() -> Result<()> { let options = CultMeshRudpDocumentServerOptions { - max_remembered_write_bytes: CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES, + max_remembered_write_bytes: 65, ..Default::default() }; let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; @@ -2806,3 +2806,65 @@ fn an_unreliable_write_on_a_legacy_session_is_dropped_silently() -> Result<()> { assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); Ok(()) } + +/// A put or delete whose document or schemaId is null or missing is refused `message_invalid` naming that +/// field (document first), with the spec's fixed text and the write's id; the session goes on. +#[test] +fn a_write_missing_its_document_or_schema_id_is_refused_naming_the_field() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 236)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + let frames = [ + ("p-missing", serde_json::json!({"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p-missing"}), "document"), + ("p-null", serde_json::json!({"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p-null", "document": null}), "document"), + ("p-noschema", serde_json::json!({"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p-noschema", + "document": {"recordKey": "k", "storedAt": "t", "payloadEncoding": "messagepack", "payload": [1]}}), "schemaId"), + ("d-null", serde_json::json!({"schemaVersion": "cultnet.document_delete.v0", "messageId": "d-null", "schemaId": null, "recordKey": "k"}), "schemaId"), + ("d-missing", serde_json::json!({"schemaVersion": "cultnet.document_delete.v0", "messageId": "d-missing", "recordKey": "k"}), "schemaId"), + ]; + for (_, frame, _) in &frames { + peer.send_reliable("schema", rmp_serde::to_vec_named(frame)?)?; + } + let answers = answers_after(&mut server, &mut peer, frames.len())?; + for ((id, _, field), answer) in frames.iter().zip(&answers) { + assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::MessageInvalid), Some((*id).into()))); + assert_eq!( + error_text_and_field(answer), + (format!("message_invalid: {field} is required"), Some((*field).into())) + ); + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// An unreliable write whose refusal the full reliable queue cannot take ends the session as a local failure +/// (the peer's write is then unconfirmed, never silent and never "refused a packet"). +#[test] +fn an_unreliable_refusal_that_cannot_be_queued_ends_the_session_as_a_local_failure() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { max_pending_reliable_packets_per_session: 3, ..Default::default() }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 237)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + // The peer acknowledges nothing, so each refusal stays in the server's reliable queue. + for index in 0..8 { + let payload = encode_cultnet_message_to_vec(&put(&format!("u{index}"), "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + } + assert_eq!(server.session_count(), 0, "the refusal the queue could not take ended the session"); + let goodbye = peer.drain().into_iter().find(is_goodbye).expect("the peer is told the session ended"); + assert_eq!(goodbye.payload, b"server could not answer a write".to_vec()); + Ok(()) +} diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 531fe7ec..0ea2aa53 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -2020,3 +2020,29 @@ pub fn readable_message_id(bytes: &[u8]) -> Option { _ => None, }) } + +/// The first required field (`document`, then `schemaId`) that a put or delete frame lacks or holds as nil +/// (write-receipt.md, "Required fields"), with the write's operation (`"put"` or `"delete"`). `None` when the +/// bytes are not such a frame, or both fields are present. A put's `schemaId` is its document's. +pub fn missing_required_write_field(bytes: &[u8]) -> Option<(&'static str, &'static str)> { + let rmpv::Value::Map(entries) = rmpv::decode::read_value(&mut &*bytes).ok()? else { + return None; + }; + let field = |entries: &[(rmpv::Value, rmpv::Value)], name: &str| { + entries + .iter() + .find(|(key, _)| key.as_str() == Some(name)) + .map(|(_, value)| value.clone()) + .filter(|value| !value.is_nil()) + }; + match field(&entries, "schemaVersion")?.as_str()? { + "cultnet.document_put_raw.v0" => match field(&entries, "document") { + Some(rmpv::Value::Map(document)) => { + field(&document, "schemaId").is_none().then_some(("put", "schemaId")) + } + _ => Some(("put", "document")), + }, + "cultnet.document_delete.v0" => field(&entries, "schemaId").is_none().then_some(("delete", "schemaId")), + _ => None, + } +} From 3443e1dc939cf9676ea1aad5b0c7126f3376f790 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:34:19 +0000 Subject: [PATCH 046/117] Spec #59: error.v1 code enum names limit_reached; server-wide budget is a conformance column; an overflowed refusal set refuses unseen ids for the session Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 5a7b90f0..c87e66ac 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. +**`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. Why a new error version and not an extra key on v0: the v0 schema is closed (`additionalProperties: false`) and the TypeScript decoder (Ajv over the closed schemas) rejects unknown keys and unknown `schemaVersion`s, and `parseCultNetMessage` throws on an unknown version; Rust's decoder errors on an unknown version too. Unknown-key tolerance of the Rust, Python and Kotlin error decoders was not checked, so none is assumed, so a surprise key or frame breaks an old client's whole receive path. Old clients are therefore never sent either new frame. This reverses the "old clients ignore it" assumption, which is false for today's decoders. @@ -35,7 +35,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **Id size.** A write's `messageId` is at most 256 bytes of UTF-8. A longer id is refused `message_invalid` (`details.field: "messageId"`, the refusal's `messageId` the id as received, text fixed per Fixed texts) BEFORE anything is remembered: no table entry, no digest, no budget charge. Such a refusal is therefore not replayed from the table; a resend is refused afresh, identically. - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. - - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and its resend is refused identically). The client mints a new id on a new session. + - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - **Reliable channel only.** A write arriving on an unreliable channel on a receipt session is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. @@ -69,12 +69,12 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Conformance status -| Server class | Advertises both frames | -|---|---| -| `CultNetDatabaseServer` | yes | -| Shard write forwarder | yes | -| Rust RUDP document server | yes | -| C# RUDP schema server (`CultNetRudpSchemaServer`, `src/GameCult.Networking/CultNetRudpSchemaServer.cs`) | no | +| Server class | Advertises both frames | Server-wide memory budget (MAY, Remembered-write limits) | +|---|---|---| +| `CultNetDatabaseServer` | yes | not implemented: only the per-session limit of 1024 applies | +| Shard write forwarder | yes | not implemented: only the per-session limit of 1024 applies | +| Rust RUDP document server | yes | implemented (32 MiB default, `max_remembered_write_bytes`) | +| C# RUDP schema server (`CultNetRudpSchemaServer`, `src/GameCult.Networking/CultNetRudpSchemaServer.cs`) | no | n/a | Advertising is per server class, not per transport. A class advertises both frames in its hello only if it implements the receipt rules end to end; the C# RUDP schema server stays "no" until it answers writes with receipts. From 2c5203dc82a11014ce9a7cbe2a07440fe12574b5 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:34:52 +0000 Subject: [PATCH 047/117] Spec #59: local_send_failed is its own unconfirmed reason Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index c87e66ac..d99959c6 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -49,7 +49,7 @@ A write settles as exactly one of: - **admitted**: a `write_admitted` whose `messageId` is the write's. - **refused**: an `error.v1` whose `messageId` is the write's, carrying its `code` and `details` as the typed error. -- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window) and `session_ended` (the session ended first, whether the peer closed it or the client's own send failed: a local send failure carries a local-failure reason, e.g. `session_ended` / `local_send_failed`). A local send failure is never reported as refused: no server answer exists, and the write may or may not have left the machine. Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. +- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window), `session_ended` (the peer closed the session, or it ended, before the write was answered) and `local_send_failed` (the client's own send of this write failed). A local send failure is never reported as refused: no server answer exists, and the write may or may not have left the machine. Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. **Result window.** Two named constants (also in the vectors): `RECEIPT_RESULT_WINDOW_MS = 5000`, the default against a receipt-capable server (its hello lists both frames); `LEGACY_RESULT_WINDOW_MS = 500`, used only as the fallback against a server whose hello does not list them (or that sent none). The window runs from the send. A receipt-capable server that has not answered within its window is reported as unconfirmed like any other; clients log it as a server that missed its window. From 1a05df2524e87f095f0e94f41a291ac259a79f21 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:36:10 +0000 Subject: [PATCH 048/117] cultmesh-rs: server-wide remembered-write budget is a maintained counter; a session whose refusal set overflowed refuses unseen ids after the budget frees Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 52 +++++-- .../cultmesh-rs/tests/rudp_document_server.rs | 137 ++++++++++++++++++ 2 files changed, 177 insertions(+), 12 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 2b320578..a06c4773 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -580,6 +580,10 @@ struct SessionEntry { /// Writes refused `limit_reached`, kept so their resend replays the refusal. A bounded set outside the /// entry budget (`MAX_REMEMBERED_WRITES_PER_SESSION`); past it a new write is refused unremembered. limit_refusals: HashMap<[u8; 32], [u8; 32]>, + /// A limit refusal was answered that `limit_refusals` had no room to remember. That id's resend cannot be + /// told from a new write, so the session refuses every id it has not seen for the rest of its life, even once + /// the server-wide budget has freed: an id refused at a bound is never admitted later. + limit_overflowed: bool, } /// What the server remembers of a write it answered: digests only, so an entry costs @@ -619,6 +623,9 @@ pub struct CultMeshRudpDocumentServer { snapshot_source: Q, clock: C, options: CultMeshRudpDocumentServerOptions, + /// Entries remembered across all live sessions (`answered_writes`), kept in step by `remember` and + /// `drop_session`, so the server-wide budget is a comparison, not a sum over sessions per write. + remembered_writes: usize, packets_dropped: u64, send_failures: u64, /// Peers whose every datagram fails to send, standing in for an unroutable @@ -655,6 +662,7 @@ where snapshot_source, clock, options, + remembered_writes: 0, packets_dropped: 0, send_failures: 0, #[cfg(test)] @@ -803,7 +811,7 @@ where }; if result.disconnected { - self.sessions.remove(&key); + self.drop_session(key); return Ok(CultMeshRudpPollOutcome::Handled); } @@ -1037,10 +1045,16 @@ where let idle_timeout_ms = duration_millis(self.options.session_idle_timeout); let lifetime_ms = duration_millis(self.options.session_max_lifetime); let before = self.sessions.len(); + let mut freed = 0; self.sessions.retain(|_, entry| { - now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms - && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms + let live = now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms + && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms; + if !live { + freed += entry.answered_writes.len(); + } + live }); + self.remembered_writes -= freed; let expired = before - self.sessions.len(); self.end_answered_sessions()?; @@ -1106,7 +1120,7 @@ where .get(&key) .is_some_and(|entry| !entry.session.connect_repeats(packet)) { - self.sessions.remove(&key); + self.drop_session(key); } let reply = match self.sessions.get_mut(&key) { Some(entry) => { @@ -1146,6 +1160,7 @@ where capability_settled: false, answered_writes: HashMap::new(), limit_refusals: HashMap::new(), + limit_overflowed: false, }, ); accept @@ -1455,7 +1470,8 @@ where /// (rebuilt from this resend, whose content hash is the original's), a repeat with other content is /// `MessageIdReused`. A first sighting past the session table or the server-wide budget is refused /// `limit_reached` for that write alone, and that refusal is remembered (`limit_refusals`) so its resend - /// replays it; no slot is freed, so the id is never admitted later in the session. + /// replays it; no slot is freed, so the id is never admitted later in the session. A refusal the bounded set had no + /// room for sets `limit_overflowed`, after which the session refuses every unseen id, budget or no budget. fn seen_write( &mut self, key: CultMeshRudpSessionKey, @@ -1476,8 +1492,8 @@ where }); let session_full = entry.answered_writes.len() >= MAX_REMEMBERED_WRITES_PER_SESSION; let refusal_room = entry.limit_refusals.len() < MAX_REMEMBERED_WRITES_PER_SESSION; - let remembered_total: usize = self.sessions.values().map(|entry| entry.answered_writes.len()).sum(); - let budget_full = remembered_total + let overflowed = entry.limit_overflowed; + let budget_full = self.remembered_writes .checked_add(1) .and_then(|entries| entries.checked_mul(CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES)) .is_none_or(|bytes| bytes > self.options.max_remembered_write_bytes); @@ -1492,10 +1508,12 @@ where self.answer_sent(key, now, &answer, rejection)? } Some(_) => Some(rejection(CultMeshRudpRejectionReason::MessageIdReused)), - None if session_full || budget_full => { - if refusal_room { - if let Some(entry) = self.sessions.get_mut(&key) { + None if session_full || budget_full || overflowed => { + if let Some(entry) = self.sessions.get_mut(&key) { + if refusal_room { entry.limit_refusals.insert(id_digest, content_hash); + } else { + entry.limit_overflowed = true; } } Some(rejection(CultMeshRudpRejectionReason::WriteLimitReached)) @@ -1506,10 +1524,20 @@ where fn remember(&mut self, key: CultMeshRudpSessionKey, id_digest: [u8; 32], content_hash: [u8; 32], answer: StoredAnswer) { if let Some(entry) = self.sessions.get_mut(&key) { - entry.answered_writes.insert(id_digest, RememberedWrite { content_hash, answer }); + if entry.answered_writes.insert(id_digest, RememberedWrite { content_hash, answer }).is_none() { + self.remembered_writes += 1; + } } } + /// Removes a session for good and gives its remembered writes back to the server-wide budget. A session taken + /// out and put back within one call is not dropped. + fn drop_session(&mut self, key: CultMeshRudpSessionKey) -> Option { + let entry = self.sessions.remove(&key)?; + self.remembered_writes -= entry.answered_writes.len(); + Some(entry) + } + /// Sends the answer owed a write. An answer that cannot be queued or sent would leave the write /// unanswered, and a different answer would contradict what the sink did; so the session ends instead, /// and the peer's write settles as unconfirmed (`session_ended`). The answer stays remembered. @@ -1637,7 +1665,7 @@ where /// that can never be sent as built is not dropped: the goodbye names its /// error instead. A refusal that cannot be encoded is not sent. fn end_session(&mut self, key: CultMeshRudpSessionKey, end: SessionEnd) -> Result<()> { - let Some(mut entry) = self.sessions.remove(&key) else { + let Some(mut entry) = self.drop_session(key) else { return Ok(()); }; let local_failure = matches!(end, SessionEnd::LocalFailure); diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 9a8c4ed3..16fc2d6b 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2785,6 +2785,143 @@ fn limit_refusals_are_remembered_in_a_bounded_set() -> Result<()> { Ok(()) } +/// A refusal the bounded set could not hold is still the session's first answer for that id: when the budget +/// frees (another session ends), the id is not admitted later in the session, and neither is any id the session +/// has not been seen to use, because the refusal that was not remembered cannot be told from a new write. +#[test] +fn a_session_whose_limit_refusals_overflowed_refuses_new_ids_after_the_budget_frees() -> Result<()> { + let clock = Clock::new(70_000); + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + session_max_lifetime: Duration::from_millis(100), + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, clock.clone(), sink.clone(), Source::default())?; + let mut holder = client(server.local_addr()?, 240)?; + connect(&mut server, &mut [&mut holder])?; + send_reliable(&mut holder, &receipt_hello())?; + send_reliable(&mut holder, &put("held", "note:held", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut holder, 2)?[1], admitted("held", "note:held"), "after the server's hello"); + + clock.set_monotonic(70_050); + let mut peer = client(server.local_addr()?, 241)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..1024 { + send_reliable(&mut peer, &put(&format!("r{index}"), "note:r", vec![1]))?; + if index % 16 == 15 { + for _ in 0..40 { + server.poll_once()?; + peer.poll_resends()?; + while peer.receive_once()?.is_some() {} + } + } + } + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + let answers = frames_after(&mut server, &mut peer, 1)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); + + clock.set_monotonic(70_101); + assert_eq!(server.maintain()?.sessions_expired, 1, "the holder ended and freed the budget"); + assert_eq!(server.session_count(), 1); + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + send_reliable(&mut peer, &put("fresh", "note:fresh", vec![1]))?; + let answers = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into())), "never admitted later"); + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::LimitReached), Some("fresh".into()))); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "only the holder's write was applied"); + Ok(()) +} + +/// A session that ends gives its remembered entries back to the server-wide budget, whichever way it ends. +#[test] +fn a_session_that_ends_frees_its_remembered_writes_from_the_budget() -> Result<()> { + let clock = Clock::new(70_000); + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + session_max_lifetime: Duration::from_millis(100), + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, clock.clone(), sink.clone(), Source::default())?; + let mut first = client(server.local_addr()?, 242)?; + connect(&mut server, &mut [&mut first])?; + send_reliable(&mut first, &receipt_hello())?; + send_reliable(&mut first, &put("a", "note:a", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut first, 2)?[1], admitted("a", "note:a"), "after the server's hello"); + + clock.set_monotonic(70_050); + let mut second = client(server.local_addr()?, 243)?; + connect(&mut server, &mut [&mut second])?; + send_reliable(&mut second, &receipt_hello())?; + send_reliable(&mut second, &put("b", "note:b", vec![1]))?; + assert_eq!(refusal_of(&frames_after(&mut server, &mut second, 2)?[1]).0, Some(CultNetErrorCode::LimitReached)); + + clock.set_monotonic(70_101); + assert_eq!(server.maintain()?.sessions_expired, 1); + send_reliable(&mut second, &put("c", "note:c", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut second, 1)?, vec![admitted("c", "note:c")], "the ended session's entry was freed"); + clock.set_monotonic(70_152); + assert_eq!(server.maintain()?.sessions_expired, 1); + assert_eq!(server.session_count(), 0); + let mut third = client(server.local_addr()?, 244)?; + connect(&mut server, &mut [&mut third])?; + send_reliable(&mut third, &receipt_hello())?; + send_reliable(&mut third, &put("d", "note:d", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut third, 2)?[1], admitted("d", "note:d"), "and so was the second's"); + Ok(()) +} + +/// Whichever way a session ends, its remembered writes go back to the budget: here the peer says goodbye, and +/// here the server ends it itself (an unreliable write whose refusal the reliable queue cannot take). +#[test] +fn a_disconnected_or_server_ended_session_frees_its_remembered_writes() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + max_pending_reliable_packets_per_session: 3, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut first = client(server.local_addr()?, 245)?; + connect(&mut server, &mut [&mut first])?; + send_reliable(&mut first, &receipt_hello())?; + send_reliable(&mut first, &put("a", "note:a", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut first, 2)?[1], admitted("a", "note:a")); + first.disconnect(Vec::new())?; + for _ in 0..20 { + server.poll_once()?; + } + assert_eq!(server.session_count(), 0, "the goodbye ended the session"); + + // The budget holds one entry. A raw peer is admitted into it, then ended by the server. + let mut second = RawPeer::connect(&mut server, 246)?; + second.send(&receipt_hello())?; + second.send(&put("b", "note:b", vec![1]))?; + server.poll_once()?; + for index in 0..8 { + let payload = encode_cultnet_message_to_vec(&put(&format!("u{index}"), "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = second.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + second.send_packet(packet)?; + } + server.poll_once()?; + } + assert_eq!(server.session_count(), 0, "the server ended the session"); + + let mut third = client(server.local_addr()?, 247)?; + connect(&mut server, &mut [&mut third])?; + send_reliable(&mut third, &receipt_hello())?; + send_reliable(&mut third, &put("c", "note:c", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut third, 2)?[1], admitted("c", "note:c"), "both ended sessions' entries were freed"); + Ok(()) +} + /// A session that never declared the capability gets no refusal for an unreliable write: it is dropped as before. #[test] fn an_unreliable_write_on_a_legacy_session_is_dropped_silently() -> Result<()> { From 0a67b4d8aa66adb03331a2cc2fcac38e372b81aa Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:42:22 +0000 Subject: [PATCH 049/117] C#: a remembered refusal keeps its code and field, never the write's strings, and replays in fixed text; a write on an unreliable channel of a receipt session is refused message_invalid (channel) Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 29 ++++++++ .../CultNetWriteReceipts.cs | 63 +++++++++++++++--- src/GameCult.Networking/Server.cs | 1 + .../CultNetWriteReceiptLoopbackTests.cs | 66 +++++++++++++++++++ .../CultNetWriteReceiptServerTests.cs | 56 +++++++++++++++- 5 files changed, 205 insertions(+), 10 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 03cfef83..31ccc428 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3580,6 +3580,35 @@ namespace GameCult.Networking } } + /// + /// A put or delete that arrived on an unreliable channel of a receipt session (write-receipt.md, Reliable channel only): + /// answered message_invalid, details channel, fixed text, carrying the write's id, and not applied. True when + /// the frame was such a write and has been answered here. + /// + internal bool RefuseUnreliableWrite(ICultNetSchemaMessage message, DeliveryMethod method) + { + if (WriteReceipts == null) return false; + if (method is DeliveryMethod.ReliableOrdered or DeliveryMethod.ReliableUnordered or DeliveryMethod.ReliableSequenced) return false; + var messageId = message switch + { + CultNetDocumentPutRawMessage put => put.MessageId, + CultNetDocumentDeleteMessage delete => delete.MessageId, + _ => null + }; + if (messageId == null && message is not (CultNetDocumentPutRawMessage or CultNetDocumentDeleteMessage)) return false; + try + { + SendSchema(CultNetErrorV1Message.From( + CultNetErrorMessage.ForMessageInvalid("channel", "receipt sessions require the reliable channel"), messageId)); + } + catch (Exception) + { + End(); + } + + return true; + } + private readonly object _writeFramingGate = new(); private bool _writeFramingSettled; diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index 7e984287..a713c771 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -70,10 +70,51 @@ namespace GameCult.Networking public static bool operator !=(CultNetDigest left, CultNetDigest right) => !left.Equals(right); } + /// + /// What a session keeps of a refusal (write-receipt.md, What is remembered): its code, the field its details name, and its + /// routing hint (a shard descriptor the server owns). Never the write's strings or an authority's reason: the replay is built + /// from fixed text and the fields of the resend, which carries the same content. + /// + internal readonly struct CultNetRememberedRefusal + { + public string? Code { get; } + public bool HasDetails { get; } + public string? Field { get; } + public CultNetShardRoutingHint? Hint { get; } + + public CultNetRememberedRefusal(string? code, bool hasDetails, string? field, CultNetShardRoutingHint? hint) + { + Code = code; HasDetails = hasDetails; Field = field; Hint = hint; + } + + public static CultNetRememberedRefusal Of(CultNetErrorMessage error) => + new(error.Code, error.Details != null, error.Details?.Field, error.RoutingHint); + + /// The refusal, in fixed text, naming the write's own record key or schema id where its details do. + public CultNetErrorMessage ToError(string schemaId, string recordKey) => new() + { + Error = Code switch + { + "unowned_schema" => "unowned_schema: no shard owns the schema and key", + null => Hint != null ? CultNetDatabaseServer.RoutingRefusalText : CultNetWriteReceiptSession.WriteFailedText, + _ => Code + }, + Code = Code, + RoutingHint = Hint, + Details = HasDetails + ? new CultNetErrorDetails + { + Field = Field, + Value = Field switch { "recordKey" => recordKey, "schemaId" => schemaId, _ => null } + } + : null + }; + } + /// /// One receipt-capable peer's writes: message id to content hash and the answer given, refusals included, so a resend of the /// same content is answered again without being decided again and the same id with other content is refused. Ids and content - /// hashes are held as fixed-size digests; the answer is held as its kind (and, for a refusal, its code, text and details), + /// hashes are held as fixed-size digests; the answer is held as its kind (and, for a refusal, its code and the field its details name), /// never the strings of the write. /// internal sealed class CultNetWriteReceiptSession @@ -100,7 +141,7 @@ namespace GameCult.Networking public CultNetDigest Hash { get; set; } public bool Abandoned { get; set; } public bool Admitted { get; set; } - public CultNetErrorMessage? Refusal { get; set; } + public CultNetRememberedRefusal? Refusal { get; set; } } public enum Begin { New, ReplayAdmitted, ReplayRefusal, Unanswered, DifferentContent, Full } @@ -111,7 +152,7 @@ namespace GameCult.Networking /// Looks up first. An id not seen is claimed for this content (); /// otherwise the outcome says what already holds it, and is the refusal to repeat. /// - public Begin TryBegin(string messageId, byte[] hash, out CultNetErrorMessage? refusal) + public Begin TryBegin(string messageId, byte[] hash, out CultNetRememberedRefusal? refusal) { var id = IdDigest(messageId); var content = CultNetDigest.Of(hash); @@ -162,7 +203,7 @@ namespace GameCult.Networking } /// The write was refused: remember the refusal so a resend repeats it. - public void Refused(string messageId, CultNetErrorMessage refusal) + public void Refused(string messageId, CultNetRememberedRefusal refusal) { lock (_gate) { @@ -270,7 +311,7 @@ namespace GameCult.Networking return false; case CultNetWriteReceiptSession.Begin.ReplayRefusal: _answered = true; - _send(CultNetErrorV1Message.From(refusal!, _messageId)); + _send(CultNetErrorV1Message.From(refusal!.Value.ToError(_schemaId, _recordKey), _messageId)); return false; case CultNetWriteReceiptSession.Begin.Unanswered: // The first attempt is still being applied and will answer this id, or its primary left it unanswered: @@ -309,9 +350,15 @@ namespace GameCult.Networking { if (_answered) return; _answered = true; - var refusal = CultNetErrorV1Message.From(error, _messageId); - if (_claimed) _session.Refused(_messageId, error); - _send(refusal); + if (_claimed) + { + // A remembered refusal is answered from what is remembered, so the replay is the refusal again. + var kept = CultNetRememberedRefusal.Of(error); + _session.Refused(_messageId, kept); + error = kept.ToError(_schemaId, _recordKey); + } + + _send(CultNetErrorV1Message.From(error, _messageId)); } /// diff --git a/src/GameCult.Networking/Server.cs b/src/GameCult.Networking/Server.cs index d9b763b9..08af0cc4 100644 --- a/src/GameCult.Networking/Server.cs +++ b/src/GameCult.Networking/Server.cs @@ -415,6 +415,7 @@ namespace GameCult.Networking { var cultNetMessage = LiteNetLibTransportConnection.DecodeSchema(frame); Logger.LogDebug($"Received CultNet schema message {cultNetMessage.SchemaVersion}"); + if (transport.RefuseUnreliableWrite(cultNetMessage, method)) return; await HandleCultNetSchemaMessageAsync(peer, user, cultNetMessage).ConfigureAwait(false); return; } diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs index 8a1f0b46..19180459 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs @@ -47,6 +47,9 @@ namespace GameCult.Networking.Tests public void Send(T message) where T : ICultNetSchemaMessage => _peer.Send(CultNetSchemaMessageSerialization.Serialize(message), DeliveryMethod.ReliableOrdered); + public void SendUnreliable(T message) where T : ICultNetSchemaMessage => + _peer.Send(CultNetSchemaMessageSerialization.Serialize(message), DeliveryMethod.Unreliable); + public void SendRaw(byte[] bytes) => _peer.Send(bytes, DeliveryMethod.ReliableOrdered); public T Next() where T : ICultNetSchemaMessage @@ -175,6 +178,69 @@ namespace GameCult.Networking.Tests Assert.That(running.Cache.Get(KeyB), Is.Null); } + // write-receipt.md, Reliable channel only: a write on an unreliable channel of a receipt session is refused + // message_invalid, details.field "channel", fixed text, carrying the write's id; it is not applied. + [Test] + public async Task AWriteOnAnUnreliableChannelOfAReceiptPeerIsRefusedAndNotApplied() + { + using var running = Running.Start(); + using var client = running.Connect(); + client.Send(HelloOf(BothFrames)); + client.Next(); + + client.SendUnreliable(running.PutOf("u1", KeyA, "lost")); + + var error = client.Next(); + Assert.That(error.Code, Is.EqualTo("message_invalid")); + Assert.That(error.Details!.Field, Is.EqualTo("channel")); + Assert.That(error.Error, Is.EqualTo("message_invalid: receipt sessions require the reliable channel")); + Assert.That(error.MessageId, Is.EqualTo("u1")); + + // The session goes on, and the same id on the reliable channel is a new write, not a replayed refusal. + client.Send(running.PutOf("u1", KeyA, "kept")); + Assert.That(client.Next().MessageId, Is.EqualTo("u1")); + await WaitFor(() => running.Cache.Get(KeyA)?.Text == "kept"); + } + + [Test] + public void ADeleteOnAnUnreliableChannelOfAReceiptPeerIsRefusedWithItsId() + { + using var running = Running.Start(); + using var client = running.Connect(); + client.Send(HelloOf(BothFrames)); + client.Next(); + + client.SendUnreliable(new CultNetDocumentDeleteMessage + { + MessageId = "ud1", SchemaId = running.SchemaId, RecordKey = KeyA.Value, ShardId = ShardId, ShardEpoch = 1 + }); + + var error = client.Next(); + Assert.That(error.Code, Is.EqualTo("message_invalid")); + Assert.That(error.Details!.Field, Is.EqualTo("channel")); + Assert.That(error.MessageId, Is.EqualTo("ud1")); + } + + [Test] + public async Task OnlyWritesAreRefusedOnTheUnreliableChannelAndALegacyPeerIsLeftAsBefore() + { + using var running = Running.Start(); + using var receipt = running.Connect(); + receipt.Send(HelloOf(BothFrames)); + receipt.Next(); + receipt.SendUnreliable(HelloOf(BothFrames)); // not a write: answered with nothing on the unreliable channel + receipt.Send(running.PutOf("r1", KeyA, "one")); + Assert.That(receipt.Next().MessageId, Is.EqualTo("r1"), "nothing else was sent ahead of the receipt"); + + using var legacy = running.Connect(); + legacy.Send(HelloOf()); + legacy.SendUnreliable(running.PutOf("l1", KeyB, "two")); // refused by the authority, answered as v0 as before + Assert.That(legacy.Next().Code, Is.EqualTo("write_not_authorized")); + legacy.SendUnreliable(running.PutOf("l2", KeyA, "three")); + await WaitFor(() => running.Cache.Get(KeyA)?.Text == "three"); + legacy.ExpectNothingMore("a legacy peer is never sent a receipt, error.v1 or a channel refusal"); + } + private static async Task WaitFor(Func condition) { var deadline = DateTime.UtcNow + Cap; diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 09188c5f..7c6537a3 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -591,7 +591,7 @@ namespace GameCult.Networking.Tests await rig.Delete(wire, "fwd-2", KeyA); AssertRefused(wire.Sent[0], "fwd-1", "write_not_authorized"); - Assert.That(((CultNetErrorV1Message)wire.Sent[0]).Error, Is.EqualTo("primary says no")); + Assert.That(((CultNetErrorV1Message)wire.Sent[0]).Error, Is.EqualTo("write_not_authorized"), "a relayed refusal is in fixed text, the same one a resend replays"); AssertRefused(wire.Sent[1], "fwd-2", "message_invalid"); } @@ -631,7 +631,7 @@ namespace GameCult.Networking.Tests await rig.Put(wire, "fwd-1", KeyA, "one"); Assert.That(wire.Sent, Has.Count.EqualTo(2)); - Assert.That(((CultNetErrorV1Message)wire.Sent[1]).Error, Is.EqualTo("primary says no")); + Assert.That(((CultNetErrorV1Message)wire.Sent[1]).Error, Is.EqualTo("write_not_authorized"), "a remembered refusal is replayed in fixed text, not the primary's own"); Assert.That(forwarder.Forwarded, Is.EqualTo(1)); } @@ -1119,5 +1119,57 @@ namespace GameCult.Networking.Tests var error = (CultNetErrorMessage)wire.Sent.Single(); Assert.That(error.Error, Is.EqualTo("the shard log request could not be answered")); } + + private static CultNetErrorMessage DeniedWith(string recordKey, string reason) => new() + { + Error = $"write_not_authorized: {reason}", + Code = "write_not_authorized", + Details = new CultNetErrorDetails { Field = "recordKey", Value = recordKey } + }; + + // write-receipt.md, What is remembered: a refusal is remembered as its code (and the field it names), never the strings of + // the write or of the authority's reason; the replay is built from fixed text and the resend's own fields. + [Test] + public void ARefusalIsReplayedFromFixedTextAndTheResendsOwnFieldsNotFromRememberedStrings() + { + var session = new CultNetWriteReceiptSession(); + var sent = new List(); + var longKey = new string('k', 50_000); + CultNetReceiptedWrite Write() => new(session, sent.Add, "put", "m1", "schema.v1", longKey, [1]); + + var first = Write(); + Assert.That(first.Begin(), Is.True); + first.Refused(DeniedWith(longKey, new string('r', 50_000))); + Assert.That(Write().Begin(), Is.False); + + var again = (CultNetErrorV1Message)sent[1]; + Assert.That(again.MessageId, Is.EqualTo("m1")); + Assert.That(again.Code, Is.EqualTo("write_not_authorized")); + Assert.That(again.Error, Is.EqualTo("write_not_authorized")); + Assert.That(again.Details!.Field, Is.EqualTo("recordKey")); + Assert.That(again.Details.Value, Is.EqualTo(longKey)); + var firstAnswer = (CultNetErrorV1Message)sent[0]; + Assert.That(firstAnswer.Error, Is.EqualTo(again.Error), "the replay is the refusal again"); + Assert.That(firstAnswer.Details!.Value, Is.EqualTo(again.Details.Value)); + } + + [Test] + public void ThousandsOfRefusalsOf50kCharacterKeysAreRememberedInBoundedMemory() + { + var session = new CultNetWriteReceiptSession(); + var before = GC.GetTotalMemory(true); + for (var i = 0; i < CultNetWriteReceiptSession.MaxRememberedWrites; i++) + { + var key = new string('k', 50_000) + i; + var write = new CultNetReceiptedWrite(session, _ => { }, "put", $"m{i}", "schema.v1", key, [1]); + Assert.That(write.Begin(), Is.True); + write.Refused(DeniedWith(key, key)); + } + + // Held as strings, the 1024 keys and reasons alone would be over 200 MB. + var growth = GC.GetTotalMemory(true) - before; + GC.KeepAlive(session); + Assert.That(growth, Is.LessThan(8L * 1024 * 1024)); + } } } From 48cf3b332368bf2d83fbe9874a6770e21ca7b966 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:45:29 +0000 Subject: [PATCH 050/117] C#: tests kill the replay-details and unreliable non-write mutants Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetWriteReceiptLoopbackTests.cs | 3 ++ .../CultNetWriteReceiptServerTests.cs | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs index 19180459..fb07ee58 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs @@ -231,6 +231,9 @@ namespace GameCult.Networking.Tests receipt.SendUnreliable(HelloOf(BothFrames)); // not a write: answered with nothing on the unreliable channel receipt.Send(running.PutOf("r1", KeyA, "one")); Assert.That(receipt.Next().MessageId, Is.EqualTo("r1"), "nothing else was sent ahead of the receipt"); + receipt.Send(running.PutOf("r2", KeyA, "two")); + Assert.That(receipt.Next().MessageId, Is.EqualTo("r2")); + receipt.ExpectNothingMore("the unreliable hello was answered with nothing"); using var legacy = running.Connect(); legacy.Send(HelloOf()); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 7c6537a3..b50ac5de 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1171,5 +1171,39 @@ namespace GameCult.Networking.Tests GC.KeepAlive(session); Assert.That(growth, Is.LessThan(8L * 1024 * 1024)); } + + // A remembered refusal is replayed with the details its code carries: the schema id of an unowned write, no details + // at all for the routing refusal. + [Test] + public void AReplayedRefusalCarriesTheDetailsOfItsCodeAndNoOthers() + { + var session = new CultNetWriteReceiptSession(); + var sent = new List(); + CultNetReceiptedWrite Write(string id) => new(session, sent.Add, "put", id, "schema.v1", "note:1", [1]); + + var unowned = Write("u1"); + Assert.That(unowned.Begin(), Is.True); + unowned.Refused(CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("schema.v1", new CultRecordKey("note:1")))); + Assert.That(Write("u1").Begin(), Is.False); + + var routed = Write("g1"); + Assert.That(routed.Begin(), Is.True); + routed.Refused(new CultNetErrorMessage + { + Error = CultNetDatabaseServer.RoutingRefusalText, + RoutingHint = new CultNetShardRoutingHint { Reason = "not_primary" } + }); + Assert.That(Write("g1").Begin(), Is.False); + + var unownedAgain = (CultNetErrorV1Message)sent[1]; + Assert.That(unownedAgain.Code, Is.EqualTo("unowned_schema")); + Assert.That(unownedAgain.Details!.Field, Is.EqualTo("schemaId")); + Assert.That(unownedAgain.Details.Value, Is.EqualTo("schema.v1")); + Assert.That(unownedAgain.Error, Is.EqualTo(((CultNetErrorV1Message)sent[0]).Error)); + var routedAgain = (CultNetErrorV1Message)sent[3]; + Assert.That(routedAgain.Details, Is.Null); + Assert.That(routedAgain.RoutingHint!.Reason, Is.EqualTo("not_primary")); + Assert.That(routedAgain.Error, Is.EqualTo(CultNetDatabaseServer.RoutingRefusalText)); + } } } From 74adad916ab25c605cd964d73dea7c6016b4f2be Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:59:39 +0000 Subject: [PATCH 051/117] Spec #59: receipt writes need a reliable, non-sequenced channel; error.v1 schema description names limit_reached; #55-#58 paragraph states which clients still window-succeed Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/cultnet.error.v1.schema.json | 2 +- contracts/cultnet/interop/write-receipt.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contracts/cultnet/interop/cultnet.error.v1.schema.json b/contracts/cultnet/interop/cultnet.error.v1.schema.json index 02a1ae21..60f24a56 100644 --- a/contracts/cultnet/interop/cultnet.error.v1.schema.json +++ b/contracts/cultnet/interop/cultnet.error.v1.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/GameCult/CultLib/contracts/cultnet/interop/cultnet.error.v1.schema.json", "title": "CultNet Error Message v1", - "description": "cultnet.error.v0 plus messageId (write-receipt.md). Sent in place of cultnet.error.v0 only to a connection that declared the write-receipt capability. v0 is unchanged: its schema is closed (additionalProperties false), so an extra key would make a v0 peer reject the frame instead of ignoring it. Map-mode: every key is written, messageId nil when the error answers no readable message. The code enum is the v0 enum.", + "description": "cultnet.error.v0 plus messageId (write-receipt.md). Sent in place of cultnet.error.v0 only to a connection that declared the write-receipt capability. v0 is unchanged: its schema is closed (additionalProperties false), so an extra key would make a v0 peer reject the frame instead of ignoring it. Map-mode: every key is written, messageId nil when the error answers no readable message. The code enum is the v0 enum plus limit_reached (write-receipt.md, Remembered-write limits).", "type": "object", "required": [ "schemaVersion", diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index d99959c6..ce708ca6 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -38,7 +38,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - - **Reliable channel only.** A write arriving on an unreliable channel on a receipt session is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. + - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. @@ -61,7 +61,7 @@ A write settles as exactly one of: ## Changes to what #55 to #58 shipped -Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The conformance work changes it; until then the window-success is the interim, and callers must not read it as admission. +Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The TypeScript one-shot writer (#62) already reports it so; the C# and Rust clients of #55 to #58 keep the window-success until their conformance work lands, and callers of those must not read it as admission. ## Vectors From e4b67a9e0bd57ade1917964b68c059a2a8fc4dd0 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 17:00:48 +0000 Subject: [PATCH 052/117] cultmesh-rs: a receipt session accepts writes only on a reliable, non-sequenced channel; a test pins that a reconnect starts a fresh session Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 9 +- .../cultmesh-rs/tests/rudp_document_server.rs | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index a06c4773..4df5cd31 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -755,10 +755,11 @@ where } if packet.packet_type == CultNetRudpPacketType::Data && packet.channel_id == "schema" - && !packet.reliable + && (!packet.reliable || (packet.sequenced && self.receipts(key))) { - // A receipt session's write that could be lost without trace cannot be answered once: it is - // refused on the reliable channel, never applied. Anything else unreliable is dropped as before. + // A receipt session's write that could be lost without trace (unreliable, or on a sequenced channel + // that drops superseded packets) cannot be answered once: it is refused, never applied. Anything + // else unreliable is dropped as before. return self.refuse_unreliable_write(key, &packet.payload); } let payload_charge = if packet.packet_type == CultNetRudpPacketType::Data { @@ -1006,7 +1007,7 @@ where }) } - /// Refuses a put or delete that arrived on an unreliable channel of a receipt session with + /// Refuses a put or delete that arrived on an unreliable or sequenced channel of a receipt session with /// `message_invalid` (`details.field` `channel`) naming its id. Nothing is remembered or applied: the id /// stays free for the same write sent reliably. fn refuse_unreliable_write( diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 16fc2d6b..ac43c21d 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2922,6 +2922,97 @@ fn a_disconnected_or_server_ended_session_frees_its_remembered_writes() -> Resul Ok(()) } +/// Receipts need a channel that is reliable and not sequenced: a reliable-sequenced write is refused like an +/// unreliable one, a reliable-unordered write is admitted, and a legacy session's sequenced write is left alone. +#[test] +fn only_reliable_non_sequenced_writes_are_accepted_on_a_receipt_session() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 248)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let send_with = |peer: &mut RawPeer, server: &mut Server, id: &str, ordered: bool, sequenced: bool| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered, sequenced, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + let mut answers = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + Ok(answers) + }; + let sequenced = send_with(&mut peer, &mut server, "s1", false, true)?; + let refusals: Vec<_> = sequenced.iter().filter(|a| matches!(a, CultNetMessage::ErrorV1 { .. })).collect(); + assert_eq!(refusals.len(), 1, "the sequenced write is answered: {sequenced:?}"); + assert_eq!(refusal_of(refusals[0]), (Some(CultNetErrorCode::MessageInvalid), Some("s1".into()))); + assert_eq!( + error_text_and_field(refusals[0]), + ("message_invalid: receipt sessions require the reliable channel".into(), Some("channel".into())) + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "nothing was applied"); + + let unordered = send_with(&mut peer, &mut server, "s1", false, false)?; + assert!(unordered.contains(&admitted("s1", "note:1")), "reliable unordered is accepted, and the id was not consumed: {unordered:?}"); + // A fresh peer: an ordered stream waits on the sequence numbers the refused packet used. + let mut ordered_peer = RawPeer::connect(&mut server, 251)?; + ordered_peer.send(&receipt_hello())?; + server.poll_once()?; + let ordered = send_with(&mut ordered_peer, &mut server, "o1", true, false)?; + assert!(ordered.contains(&admitted("o1", "note:1")), "reliable ordered is accepted: {ordered:?}"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); + + let legacy_sink = Sink::default(); + let mut legacy_server: Server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + legacy_sink.clone(), + Source::default(), + Clock::new(70_000), + Default::default(), + )?; + let mut legacy = RawPeer::connect(&mut legacy_server, 249)?; + let none = send_with(&mut legacy, &mut legacy_server, "l1", false, true)?; + assert!(none.is_empty(), "a legacy session is sent nothing: {none:?}"); + assert_eq!(legacy_sink.0.lock().unwrap().receipts.len(), 1, "and its reliable-sequenced write is applied as before"); + Ok(()) +} + +/// A Connect that does not repeat the accepted one is a new client session: it starts with nothing remembered, so +/// the id a receipt session already used is a new write on it, and the old session's entries leave the budget. +#[test] +fn a_reconnect_is_a_new_receipt_session_that_remembers_nothing_of_the_old() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { max_remembered_write_bytes: 65, ..Default::default() }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let target = server.local_addr()?; + let shared = UdpSocket::bind("127.0.0.1:0")?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, target, 250, 50_000)?; + connect(&mut server, &mut [&mut first])?; + send_reliable(&mut first, &receipt_hello())?; + send_reliable(&mut first, &put("m1", "note:a", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut first, 2)?[1], admitted("m1", "note:a")); + + let mut restarted = client_on(twin, target, 250, 7)?; + connect(&mut server, &mut [&mut restarted])?; + assert_eq!(server.session_count(), 1); + send_reliable(&mut restarted, &receipt_hello())?; + send_reliable(&mut restarted, &put("m1", "note:b", vec![2]))?; + let answers = frames_after(&mut server, &mut restarted, 2)?; + assert_eq!(answers[1], admitted("m1", "note:b"), "same id, other content: a new write on a new session, within a one-entry budget"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); + Ok(()) +} + /// A session that never declared the capability gets no refusal for an unreliable write: it is dropped as before. #[test] fn an_unreliable_write_on_a_legacy_session_is_dropped_silently() -> Result<()> { From 40b34295e33038177eb973ca2deab225cc98352a Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 17:01:59 +0000 Subject: [PATCH 053/117] C#: receipt writes need a reliable, non-sequenced channel; a refusal without a record key or schema id carries a null detail value Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 4 +-- .../CultNetWriteReceipts.cs | 4 ++- .../CultNetWriteReceiptLoopbackTests.cs | 33 +++++++++++++++++++ .../CultNetWriteReceiptServerTests.cs | 20 +++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 31ccc428..fc41b2f4 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3581,14 +3581,14 @@ namespace GameCult.Networking } /// - /// A put or delete that arrived on an unreliable channel of a receipt session (write-receipt.md, Reliable channel only): + /// A put or delete that arrived on an unreliable or sequenced channel of a receipt session (write-receipt.md, Reliable channel only): /// answered message_invalid, details channel, fixed text, carrying the write's id, and not applied. True when /// the frame was such a write and has been answered here. /// internal bool RefuseUnreliableWrite(ICultNetSchemaMessage message, DeliveryMethod method) { if (WriteReceipts == null) return false; - if (method is DeliveryMethod.ReliableOrdered or DeliveryMethod.ReliableUnordered or DeliveryMethod.ReliableSequenced) return false; + if (method is DeliveryMethod.ReliableOrdered or DeliveryMethod.ReliableUnordered) return false; var messageId = message switch { CultNetDocumentPutRawMessage put => put.MessageId, diff --git a/src/GameCult.Networking/CultNetWriteReceipts.cs b/src/GameCult.Networking/CultNetWriteReceipts.cs index a713c771..2a90f35d 100644 --- a/src/GameCult.Networking/CultNetWriteReceipts.cs +++ b/src/GameCult.Networking/CultNetWriteReceipts.cs @@ -90,6 +90,8 @@ namespace GameCult.Networking public static CultNetRememberedRefusal Of(CultNetErrorMessage error) => new(error.Code, error.Details != null, error.Details?.Field, error.RoutingHint); + private static string? NullIfEmpty(string value) => value.Length == 0 ? null : value; + /// The refusal, in fixed text, naming the write's own record key or schema id where its details do. public CultNetErrorMessage ToError(string schemaId, string recordKey) => new() { @@ -105,7 +107,7 @@ namespace GameCult.Networking ? new CultNetErrorDetails { Field = Field, - Value = Field switch { "recordKey" => recordKey, "schemaId" => schemaId, _ => null } + Value = Field switch { "recordKey" => NullIfEmpty(recordKey), "schemaId" => NullIfEmpty(schemaId), _ => null } } : null }; diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs index fb07ee58..d5e32fa0 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptLoopbackTests.cs @@ -50,6 +50,9 @@ namespace GameCult.Networking.Tests public void SendUnreliable(T message) where T : ICultNetSchemaMessage => _peer.Send(CultNetSchemaMessageSerialization.Serialize(message), DeliveryMethod.Unreliable); + public void Send(T message, DeliveryMethod method) where T : ICultNetSchemaMessage => + _peer.Send(CultNetSchemaMessageSerialization.Serialize(message), method); + public void SendRaw(byte[] bytes) => _peer.Send(bytes, DeliveryMethod.ReliableOrdered); public T Next() where T : ICultNetSchemaMessage @@ -202,6 +205,36 @@ namespace GameCult.Networking.Tests await WaitFor(() => running.Cache.Get(KeyA)?.Text == "kept"); } + // write-receipt.md, Reliable channel only: reliable and not sequenced. Reliable ordered and unordered qualify; reliable + // sequenced does not, and nor does any unreliable method. + [TestCase(DeliveryMethod.ReliableSequenced, true)] + [TestCase(DeliveryMethod.Sequenced, true)] + [TestCase(DeliveryMethod.Unreliable, true)] + [TestCase(DeliveryMethod.ReliableUnordered, false)] + [TestCase(DeliveryMethod.ReliableOrdered, false)] + public async Task OnlyAReliableNonSequencedChannelCarriesAWriteOfAReceiptPeer(DeliveryMethod method, bool refused) + { + using var running = Running.Start(); + using var client = running.Connect(); + client.Send(HelloOf(BothFrames)); + client.Next(); + + client.Send(running.PutOf("c1", KeyA, "text"), method); + + if (refused) + { + var error = client.Next(); + Assert.That(error.Details!.Field, Is.EqualTo("channel")); + Assert.That(error.MessageId, Is.EqualTo("c1")); + Assert.That(running.Cache.Get(KeyA), Is.Null); + } + else + { + Assert.That(client.Next().MessageId, Is.EqualTo("c1")); + await WaitFor(() => running.Cache.Get(KeyA)?.Text == "text"); + } + } + [Test] public void ADeleteOnAnUnreliableChannelOfAReceiptPeerIsRefusedWithItsId() { diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index b50ac5de..6a79db6e 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1205,5 +1205,25 @@ namespace GameCult.Networking.Tests Assert.That(routedAgain.RoutingHint!.Reason, Is.EqualTo("not_primary")); Assert.That(routedAgain.Error, Is.EqualTo(CultNetDatabaseServer.RoutingRefusalText)); } + + // A write that carries no record key or schema id has none to echo: the detail value is null, not an empty string. + [Test] + public void AReplayedRefusalOfAWriteWithoutTheNamedFieldCarriesANullValue() + { + var session = new CultNetWriteReceiptSession(); + var sent = new List(); + CultNetReceiptedWrite Write() => new(session, sent.Add, "delete", "m1", null, null, null); + var first = Write(); + Assert.That(first.Begin(), Is.True); + first.Refused(DeniedWith("ignored", "reason")); + Assert.That(Write().Begin(), Is.False); + + foreach (var message in sent) + { + var error = (CultNetErrorV1Message)message; + Assert.That(error.Details!.Field, Is.EqualTo("recordKey")); + Assert.That(error.Details.Value, Is.Null); + } + } } } From 34bf7aa0ee6490da63e8828355f2314ec090e244 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 17:03:36 +0000 Subject: [PATCH 054/117] C#: test kills the schema id detail-value mutant Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetWriteReceiptServerTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 6a79db6e..13aab811 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1224,6 +1224,12 @@ namespace GameCult.Networking.Tests Assert.That(error.Details!.Field, Is.EqualTo("recordKey")); Assert.That(error.Details.Value, Is.Null); } + + var unowned = new CultNetReceiptedWrite(session, sent.Add, "delete", "m2", null, null, null); + Assert.That(unowned.Begin(), Is.True); + unowned.Refused(CultNetErrorMessage.ForUnownedSchema(new CultNetUnownedSchemaException("x", new CultRecordKey("k")))); + Assert.That(((CultNetErrorV1Message)sent[^1]).Details!.Field, Is.EqualTo("schemaId")); + Assert.That(((CultNetErrorV1Message)sent[^1]).Details!.Value, Is.Null); } } } From 6eea66c57f18b6e6a753c248f673c03e5ff0c699 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 14:59:59 +0000 Subject: [PATCH 055/117] cultnet-ts, cultmesh-ts: write receipt client (frames, WriteTracker, replay runner, opt-in writer) Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/CHANGELOG.md | 4 + packages/cultmesh-ts/src/index.ts | 95 +++++++-- .../test/write-receipt-client.test.ts | 139 ++++++++++++++ packages/cultnet-ts/CHANGELOG.md | 1 + packages/cultnet-ts/README.md | 27 +++ packages/cultnet-ts/src/contracts.ts | 43 +++++ packages/cultnet-ts/src/index.ts | 18 ++ packages/cultnet-ts/src/write-tracker.ts | 133 +++++++++++++ .../cultnet-ts/test/write-receipt.test.ts | 181 ++++++++++++++++++ .../tools/replay-write-receipt-vectors.mjs | 116 +++++++++++ 10 files changed, 740 insertions(+), 17 deletions(-) create mode 100644 packages/cultmesh-ts/test/write-receipt-client.test.ts create mode 100644 packages/cultnet-ts/src/write-tracker.ts create mode 100644 packages/cultnet-ts/test/write-receipt.test.ts create mode 100644 packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index d06e753b..f803476a 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following ## Unreleased +### Added + +- `publishRudpDocumentOnce` takes `writeReceipts: true`: it sends a hello listing `cultnet.write_admitted.v0` and `cultnet.error.v1` and, against a server that lists both back, resolves only on that write's `write_admitted`, rejects with `CultMeshRudpWriteRefusedError` (now also carrying an `error.v1`) on its refusal and with the new `CultMeshRudpWriteUnconfirmedError` (`reason`: `window_elapsed` after 5000 ms, or `session_ended`) when nothing settles it. Without the option, or against a server that did not advertise, nothing changes: no hello, a 500 ms window, and a window with no refusal still reads as published (interim; not admission). `resultWindowMs` overrides either window. `publishRudpDocumentAndWaitForReceipt` is not changed. + ### Breaking - The package is renamed `cultmesh-ts` to `@gamecult/cultmesh-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-ts` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`. diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index f065dd83..eab2e6dc 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -20,6 +20,10 @@ import { CultNetRudpSocketTransportConnection, rudpClientBindHost, CultNetSchemaCatalog, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + WriteTracker, + helloAdvertisesWriteReceipts, + resultWindowMs, cultNetMessageInvalidError, cultNetTextError, cultNetWriteNotAuthorizedError, @@ -44,6 +48,8 @@ import { type CultNetSchemaCatalogOptions, type CultNetSnapshotResponseRawMessage, type CultNetWireContract, + type WriteSettlement, + type WriteUnconfirmedReason, } from "@gamecult/cultnet-ts"; import { @@ -4189,8 +4195,8 @@ export interface CultMeshRudpDocumentServer { */ export class CultMeshRudpWriteRefusedError extends Error { readonly code: string | undefined; - readonly response: Extract; - constructor(response: Extract) { + readonly response: Extract; + constructor(response: Extract) { super(response.error); this.name = "CultMeshRudpWriteRefusedError"; this.code = response.code ?? undefined; @@ -4198,14 +4204,40 @@ export class CultMeshRudpWriteRefusedError extends Error { } } +/** + * Nothing settled the write (write-receipt.md, "unconfirmed"): the result window ran out with no answer, or the + * session ended first. Not success and not refusal; a caller that needs admission treats it as not admitted. + */ +export class CultMeshRudpWriteUnconfirmedError extends Error { + readonly reason: WriteUnconfirmedReason; + readonly messageId: string; + constructor(messageId: string, reason: WriteUnconfirmedReason) { + super(`Write ${messageId} was not confirmed: ${reason}.`); + this.name = "CultMeshRudpWriteUnconfirmedError"; + this.reason = reason; + this.messageId = messageId; + } +} + export interface CultMeshRudpDocumentPublishOptions extends CultMeshRudpPeerOptions { /** - * How long after the put's transport ack a refusal may still arrive. A reliable-transport ack is not - * admission; the outcome is the peer's result frame. A server that answers a put only on refusal sends no - * success frame, so the put is reported published once this window passes with no refusal; a success reply - * ends the wait early. Default 500 ms. + * How long, counted from the send, the put may wait for its result frame. A reliable-transport ack is not + * admission; the outcome is the peer's result frame. Default: 5000 ms against a server whose hello lists the + * write-receipt frames, 500 ms otherwise. Against a server that did not advertise them the put is reported + * published once the window passes with no refusal (interim; the caller must not read it as admission), and + * a success reply ends the wait early. */ resultWindowMs?: number; + /** + * Declare the write-receipt capability: send a hello listing `cultnet.write_admitted.v0` and `cultnet.error.v1` + * before the put. A server that lists both back is receipt-capable: the put then resolves only on its + * `write_admitted`, rejects with `CultMeshRudpWriteRefusedError` on its `error.v1`, and rejects with + * `CultMeshRudpWriteUnconfirmedError` when nothing settles it. Unset, no hello is sent and the server answers + * as it always did. + */ + writeReceipts?: boolean; + /** With `writeReceipts`, how long to wait for the server's hello before treating it as legacy. Default 500 ms. */ + helloWaitMs?: number; messageId?: string; sourceRuntimeId?: string; sourceAgentId?: string; @@ -6182,17 +6214,42 @@ export class CultMesh { const messageId = options.messageId ?? `${runtimeId}:${binding.definition.type}:${recordKey}`; const peer = await CultMesh.createRudpPeer(runtimeId, connectionId, endpoint, options); try { - // A transport ack is not admission: the put completes on its result frame. A refusal rejects with the - // typed error whether it arrives before the ack, after it, or inside the result window. - let refusal: CultMeshRudpWriteRefusedError | undefined; - let succeeded = false; + // A transport ack is not admission: the put completes on its result frame, and the WriteTracker alone decides + // how it settles (write-receipt.md, "Client rules"). This function only maps the settlement to resolve/throw. + let receiptCapable = false; + let tracker: WriteTracker | undefined; + let settlement: WriteSettlement | undefined; let wake: (() => void) | undefined; + let serverHello: (() => void) | undefined; peer.on("message", (message: CultNetMessage) => { - if (message.schemaVersion === "cultnet.error.v0") refusal ??= new CultMeshRudpWriteRefusedError(message); - else if ("messageId" in message && message.messageId === messageId) succeeded = true; - else return; - wake?.(); + if (message.schemaVersion === "cultnet.hello.v0") { + receiptCapable = helloAdvertisesWriteReceipts(message); + serverHello?.(); + } else if (!receiptCapable && "messageId" in message && message.messageId === messageId + && message.schemaVersion !== "cultnet.error.v1") { + // A server that did not advertise receipts may echo the id back as its success reply. + tracker?.frame({ schemaVersion: "cultnet.write_admitted.v0", messageId, operation: "put", schemaId: binding.definition.schemaId ?? "", recordKey }, Date.now()); + } else { + tracker?.frame(message, Date.now()); + } }); + peer.on("close", () => tracker?.sessionEnd(Date.now())); + if (options.writeReceipts) { + const answered = new Promise((resolve) => { serverHello = resolve; }); + peer.sendHello({ + schemaVersion: "cultnet.hello.v0", runtimeId, runtimeKind: "cultmesh-ts", + supportedMessageVersions: [...CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS], + }); + let helloTimer: NodeJS.Timeout | undefined; + await Promise.race([answered, new Promise((resolve) => { helloTimer = setTimeout(resolve, options.helloWaitMs ?? 500); })]); + clearTimeout(helloTimer); + } + const writeTracker = new WriteTracker({ + windowMs: options.resultWindowMs ?? resultWindowMs(receiptCapable), + onSettle: (_id, result) => { settlement = result; wake?.(); }, + }); + tracker = writeTracker; + writeTracker.send(messageId, Date.now()); peer.send(registry.createRawDocumentPutMessage( binding, messageId, @@ -6206,13 +6263,17 @@ export class CultMesh { }, )); await peer.flush(options.flushTimeoutMs ?? 30_000); - if (!refusal && !succeeded) { + while (!settlement) { + const remaining = writeTracker.nextDeadlineMs(Date.now()) ?? 0; await new Promise((resolve) => { - const timer = setTimeout(resolve, options.resultWindowMs ?? 500); + const timer = setTimeout(resolve, remaining); wake = () => { clearTimeout(timer); resolve(); }; }); + writeTracker.tick(Date.now()); } - if (refusal) throw refusal; + if (settlement.outcome === "refused") throw new CultMeshRudpWriteRefusedError(settlement.response); + // Interim, for a server that did not advertise receipts: a window with no refusal reads as published. + if (settlement.outcome === "unconfirmed" && receiptCapable) throw new CultMeshRudpWriteUnconfirmedError(messageId, settlement.reason); } finally { peer.close(); } diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts new file mode 100644 index 00000000..ddc3d41e --- /dev/null +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -0,0 +1,139 @@ +// Client side of contracts/cultnet/interop/write-receipt.md in the one-shot writer: with `writeReceipts` the put +// completes only on a write_admitted carrying its id; a refusal, a silent window and a legacy server each end as the +// spec says. Each test serves the client a fake RUDP peer that acknowledges the put and answers as scripted. +import assert from "node:assert/strict"; +import { createSocket, type Socket } from "node:dgram"; +import test from "node:test"; +import { z } from "zod"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + CultNetRudpSession, + decodeRudpPacket, + defineCultNetDocumentBinding, + encodeCultNetMessageForWire, + encodeRudpPacket, + type CultNetMessage, +} from "@gamecult/cultnet-ts"; +import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; +import { CultMesh, CultMeshRudpWriteRefusedError, CultMeshRudpWriteUnconfirmedError } from "../src/index"; + +const note = defineDocumentType({ + type: "cultmesh.receipt_note", + schemaId: "cultmesh.receipt_note.v0", + schema: z.object({ noteId: z.string(), text: z.string() }), + name: "noteId", +}); +const binding = defineCultNetDocumentBinding({ definition: note }); +let nextConnection = 0x10206000; + +interface Script { + /** Does the server's hello list both receipt frames? */ + advertise: boolean; + /** What the server answers a put with, `delayMs` after it arrives. */ + answer: "admit" | "refuse" | "wrong-id" | "silence"; + delayMs?: number; +} + +async function fakeServer(connectionId: number, script: Script) { + const socket: Socket = createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + const session = new CultNetRudpSession({ connectionId, resendDelayMs: 50 }); + const timers = new Set(); + const seen: string[] = []; + let closed = false; + const send = (packet: Parameters[0], port: number, address: string) => { + if (!closed) socket.send(encodeRudpPacket(packet), port, address); + }; + const say = (message: CultNetMessage, port: number, address: string) => { + const packets = session.sendMany("schema", encodeWireValue(encodeCultNetMessageForWire(message)), { reliable: true, ordered: true, nowMs: Date.now() }); + packets.forEach((p) => send(p, port, address)); + }; + socket.on("message", (wire, remote) => { + const packet = decodeRudpPacket(wire); + const now = Date.now(); + if (packet.packetType === "connect") { + send(session.acceptConnect(packet, now), remote.port, remote.address); + return; + } + const result = session.receive(packet, now); + if (result.reply) send(result.reply, remote.port, remote.address); + for (const frame of result.delivered) { + if (frame.channelId !== "schema") continue; + const message = decodeWireValue(frame.payload) as { schemaVersion: string; messageId?: string }; + seen.push(message.schemaVersion); + if (message.schemaVersion === "cultnet.hello.v0") { + say({ + schemaVersion: "cultnet.hello.v0", runtimeId: "fake", runtimeKind: "fake", + ...(script.advertise ? { supportedMessageVersions: [...CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS] } : {}), + }, remote.port, remote.address); + continue; + } + if (message.schemaVersion !== "cultnet.document_put_raw.v0" || script.answer === "silence") continue; + const timer = setTimeout(() => { + timers.delete(timer); + if (script.answer === "refuse") { + say({ schemaVersion: "cultnet.error.v1", error: "write_not_authorized", routingHint: null, code: "write_not_authorized", details: null, messageId: message.messageId! }, remote.port, remote.address); + } else { + const messageId = script.answer === "wrong-id" ? "someone-else" : message.messageId!; + say({ schemaVersion: "cultnet.write_admitted.v0", messageId, operation: "put", schemaId: "cultmesh.receipt_note.v0", recordKey: "tests:receipt:a" }, remote.port, remote.address); + } + }, script.delayMs ?? 20); + timers.add(timer); + } + if (packet.reliable) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); + }); + return { endpoint: `rudp://127.0.0.1:${socket.address().port}`, seen, close: () => { closed = true; timers.forEach(clearTimeout); socket.close(); } }; +} + +const transport = { resendDelayMs: 50, resendPollMs: 5, flushTimeoutMs: 2_000 }; +const publish = (server: { endpoint: string }, options: Record) => + CultMesh.publishRudpDocumentOnce("receipt-client", nextConnection++, server.endpoint, binding, "tests:receipt:a", { noteId: "a", text: "x" }, { ...transport, ...options }); + +test("a receipt-capable server's write_admitted completes the put, past the legacy 500 ms window", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "admit", delayMs: 700 }); + try { + await publish(server, { writeReceipts: true }); + assert.deepEqual(server.seen.slice(0, 2), ["cultnet.hello.v0", "cultnet.document_put_raw.v0"]); + } finally { server.close(); } +}); + +test("a receipt-capable server's error.v1 refuses the put with its code", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "refuse" }); + try { + await assert.rejects(publish(server, { writeReceipts: true }), (error: unknown) => + error instanceof CultMeshRudpWriteRefusedError && error.code === "write_not_authorized" && error.response.schemaVersion === "cultnet.error.v1"); + } finally { server.close(); } +}); + +test("a receipt-capable server that stays silent leaves the put unconfirmed, never published", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "silence" }); + try { + await assert.rejects(publish(server, { writeReceipts: true, resultWindowMs: 250 }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed"); + } finally { server.close(); } +}); + +test("a receipt for another id settles nothing", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "wrong-id" }); + try { + await assert.rejects(publish(server, { writeReceipts: true, resultWindowMs: 250 }), CultMeshRudpWriteUnconfirmedError); + } finally { server.close(); } +}); + +test("a server whose hello does not advertise keeps the interim 500 ms window-success", async () => { + const server = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); + const started = Date.now(); + try { + await publish(server, { writeReceipts: true }); + assert.ok(Date.now() - started >= 500, "the legacy window ran from the send"); + } finally { server.close(); } +}); + +test("without writeReceipts no hello is sent and the receipt frames are never asked for", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "silence" }); + try { + await publish(server, { resultWindowMs: 100 }); + assert.deepEqual(server.seen, ["cultnet.document_put_raw.v0"]); + } finally { server.close(); } +}); diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index 739396cd..13dc3dff 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@gamecult/cultnet-ts` are recorded here, following ### Added +- Write receipt, client side (`contracts/cultnet/interop/write-receipt.md`): `cultnet.write_admitted.v0` and `cultnet.error.v1` (v0 plus `messageId`; its closed code set adds `limit_reached`, `CULTNET_ERROR_V1_CODES`) parse and encode, and `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS` / `helloAdvertisesWriteReceipts` name the hello capability. `cultnet.error.v0` and its code set are unchanged. `WriteTracker` settles each write as admitted, refused or unconfirmed (windows `RECEIPT_RESULT_WINDOW_MS` 5000 and `LEGACY_RESULT_WINDOW_MS` 500, counted from the send; late answers recorded as `admitted_late` / `refused_late`), and `tools/replay-write-receipt-vectors.mjs` replays the vectors against it (see README, "Write receipt replay runner"). The interop schemas are read from `contracts/cultnet/interop/`. - `message_invalid` joins the closed `cultnet.error.v0` code set, with `cultNetMessageInvalidError(field, reason)` (`details.field` names the request field at fault), byte-identical to the C# `ForMessageInvalid`. - `cultNetWriteNotAuthorizedError(recordKey, reason?)`: the typed `write_not_authorized` refusal (code and `details.recordKey`), byte-identical to the C# `ForRemoteWriteRefused` vector in `error-vectors.json`. `CultNetDocumentRegistry.canonicalSchemaId(schemaId)` resolves the id a put or delete writes under, for a server's write authority. - `cultNetTextError(error)`: the text-only `cultnet.error.v0` refusal, with `routingHint`, `code` and `details` null. The legacy-union error decode uses it, so there is one place that spells the null members. diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index 1fb37a3f..07996956 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -289,6 +289,33 @@ maps them back to the exact C# union payload shape at the wire boundary. serve and dial commands also exercise schema-v0 over the shared RUDP socket transport and advertise both transport profiles. +## Write receipt replay runner + +`WriteTracker` is the client half of `contracts/cultnet/interop/write-receipt.md`: a pure state machine on an abstract +clock that owns one decision, how each sent write settles. `tools/replay-write-receipt-vectors.mjs` replays +`write-receipt-vectors.json` against it (build first, then `node tools/replay-write-receipt-vectors.mjs`): it prints +`PASS`/`FAIL` per case and exits nonzero on any failure. Rust and C# port the same runner by implementing this interface +and replaying the same file. + +Tracker interface (times are integer milliseconds on the vector's clock; the constructor takes the window): + +| Call | Meaning | +| --- | --- | +| `new WriteTracker({ windowMs, onSettle })` | `windowMs` = 5000 when the server's hello lists both `cultnet.write_admitted.v0` and `cultnet.error.v1`, else 500. `onSettle(id, settlement)` fires once per write. | +| `send(id, atMs)` | A write with that `messageId` was sent. Its window runs from here, not from any transport ack. | +| `frame(message, atMs)` | A decoded frame arrived. `write_admitted` and `error.v1` settle the write they name; `error.v0`, or `error.v1` with nil `messageId`, refuses a write only when exactly one is unanswered; an id not in flight settles nothing. | +| `tick(atMs)` | Advance the clock. Every unanswered write with `atMs - sentAt >= windowMs` settles `unconfirmed(window_elapsed)`. `send`, `frame` and `sessionEnd` tick first. | +| `sessionEnd(atMs)` | Every unanswered write settles `unconfirmed(session_ended)`. | +| `state(id)` | `pending`, or the settlement plus `recorded`: `admitted_late` / `refused_late`, set by the first answer that arrives after `unconfirmed`. A settlement never changes. | + +Runner, per case in `scenarios.cases` (the transport ack is not an input: delivery is not an answer): + +1. The fake server states its capability the only way a real one does, in a hello (`server: "receipt"` lists both frames, `"legacy"` lists none); the window comes from that hello and must equal `constants[case.window]`. +2. Events run in `atMs` order: `send` -> `send(id)`, `frame` -> encode the frame to MessagePack, decode and validate it as the client would, then `frame(message)`, `tick` -> `tick`, `sessionEnd` -> `sessionEnd`, `transportAck` -> nothing. +3. For each id in `expect`, the settlement given to the caller has the expected `outcome` (and `code` or `reason` when given), `state(id).recorded` equals the expected `recorded` (absent when absent), and nothing settled twice or outside `expect`. + +The runner also decodes each `wire[].messagePackHex` and compares the fields and the re-encoded bytes to the vector, and requires every `schemaInvalid[]` frame to be refused by the decoder. `dedupe.cases` and `contentHash` are server vectors and not replayed here. + ## Development ```bash diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index d4bc1b19..6e05c9a5 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -9,6 +9,8 @@ import registerSchema from "../contracts/cultnet/cultnet.register.schema.json"; import verifySchema from "../contracts/cultnet/cultnet.verify.schema.json"; import loginSuccessSchema from "../contracts/cultnet/cultnet.login-success.schema.json"; import errorSchema from "../contracts/cultnet/cultnet.error.schema.json"; +import errorV1Schema from "../contracts/cultnet/interop/cultnet.error.v1.schema.json"; +import writeAdmittedSchema from "../contracts/cultnet/interop/cultnet.write-admitted.schema.json"; import sampleChangeNameSchema from "../contracts/cultnet/cultnet.sample-change-name.schema.json"; import sampleChatSchema from "../contracts/cultnet/cultnet.sample-chat.schema.json"; import documentPutSchema from "../contracts/cultnet/cultnet.document-put.schema.json"; @@ -62,6 +64,8 @@ export type CultNetSchemaVersion = | "cultnet.verify.v0" | "cultnet.login_success.v0" | "cultnet.error.v0" + | "cultnet.error.v1" + | "cultnet.write_admitted.v0" | "cultnet.sample.change_name.v0" | "cultnet.sample.chat.v0" | "cultnet.document_put.v0" @@ -201,6 +205,11 @@ export const CULTNET_ERROR_CODES = [ export type CultNetErrorCode = (typeof CULTNET_ERROR_CODES)[number]; +/** The closed refusal-code set of cultnet.error.v1 (contracts/cultnet/interop/cultnet.error.v1.schema.json): v0's codes plus `limit_reached`. */ +export const CULTNET_ERROR_V1_CODES = [...CULTNET_ERROR_CODES, "limit_reached"] as const; + +export type CultNetErrorV1Code = (typeof CULTNET_ERROR_V1_CODES)[number]; + /** A refusal that carries text only: `routingHint`, `code` and `details` are written as null, as the reference writes an absent member. */ export function cultNetTextError(error: string): CultNetErrorMessage { return { schemaVersion: "cultnet.error.v0", error, routingHint: null, code: null, details: null }; @@ -255,6 +264,34 @@ export interface CultNetErrorMessage { details: CultNetErrorDetails | null; } +/** + * cultnet.error.v0 plus the `messageId` of the request it answers (nil when no id could be read). Sent only to a + * connection whose hello listed both receipt frames (contracts/cultnet/interop/write-receipt.md). + */ +export interface CultNetErrorV1Message extends Omit { + schemaVersion: "cultnet.error.v1"; + code: CultNetErrorV1Code | null; + messageId: string | null; +} + +/** The receipt for one admitted put or delete: every key written, the write's `messageId` echoed byte for byte. */ +export interface CultNetWriteAdmittedMessage { + schemaVersion: "cultnet.write_admitted.v0"; + messageId: string; + operation: "put" | "delete"; + schemaId: string; + recordKey: string; +} + +/** The two frames a receipt-capable hello lists in `supportedMessageVersions`; a peer lists both or neither. */ +export const CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS = ["cultnet.write_admitted.v0", "cultnet.error.v1"] as const; + +/** True when a hello lists both receipt frames: the only thing that makes a connection receipt-capable. */ +export function helloAdvertisesWriteReceipts(hello: Pick): boolean { + const listed = hello.supportedMessageVersions ?? []; + return CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS.every((version) => listed.includes(version)); +} + export interface CultNetSampleChangeNameMessage { schemaVersion: "cultnet.sample.change_name.v0"; name: string; @@ -770,6 +807,8 @@ export interface GhostlightAgentStateDocument { export type CultNetMessage = | CultNetHelloMessage + | CultNetErrorV1Message + | CultNetWriteAdmittedMessage | CultNetLoginMessage | CultNetRegisterMessage | CultNetVerifyMessage @@ -812,6 +851,8 @@ const CULTNET_MESSAGE_SCHEMAS = [ verifySchema, loginSuccessSchema, errorSchema, + errorV1Schema, + writeAdmittedSchema, sampleChangeNameSchema, sampleChatSchema, documentPutSchema, @@ -1535,6 +1576,8 @@ export const cultNetSchemas = { verifySchema, loginSuccessSchema, errorSchema, + errorV1Schema, + writeAdmittedSchema, sampleChangeNameSchema, sampleChatSchema, documentPutSchema, diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 4fd0014c..a8740cae 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -17,6 +17,12 @@ export { type CultNetRawDocumentRecord, type CultNetRawPayloadEncoding, CULTNET_ERROR_CODES, + CULTNET_ERROR_V1_CODES, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + helloAdvertisesWriteReceipts, + type CultNetErrorV1Code, + type CultNetErrorV1Message, + type CultNetWriteAdmittedMessage, cultNetTextError, cultNetMessageInvalidError, cultNetWriteNotAuthorizedError, @@ -249,3 +255,15 @@ export { type IdunnRuntimePresencePublisher, type IdunnRuntimePresencePublisherOptions, } from "./idunn-odin-presence-publisher"; + +export { + LEGACY_RESULT_WINDOW_MS, + RECEIPT_RESULT_WINDOW_MS, + WriteTracker, + resultWindowMs, + type WriteRecorded, + type WriteSettlement, + type WriteState, + type WriteTrackerOptions, + type WriteUnconfirmedReason, +} from "./write-tracker"; diff --git a/packages/cultnet-ts/src/write-tracker.ts b/packages/cultnet-ts/src/write-tracker.ts new file mode 100644 index 00000000..7398afaa --- /dev/null +++ b/packages/cultnet-ts/src/write-tracker.ts @@ -0,0 +1,133 @@ +// The client half of the write receipt (contracts/cultnet/interop/write-receipt.md, "Client rules"): a pure state +// machine with an abstract clock. It owns exactly one decision, how each sent write settles. Transport, timers and +// frame decoding belong to the caller; the replay runner (tools/replay-write-receipt-vectors.mjs) drives this same +// interface from write-receipt-vectors.json. + +import type { CultNetErrorMessage, CultNetErrorV1Message, CultNetMessage } from "./contracts"; + +/** The result window against a server whose hello lists both receipt frames. */ +export const RECEIPT_RESULT_WINDOW_MS = 5000; +/** The result window against a server whose hello does not list them (or that sent none). */ +export const LEGACY_RESULT_WINDOW_MS = 500; + +export function resultWindowMs(serverAdvertisesReceipts: boolean): number { + return serverAdvertisesReceipts ? RECEIPT_RESULT_WINDOW_MS : LEGACY_RESULT_WINDOW_MS; +} + +export type WriteUnconfirmedReason = "window_elapsed" | "session_ended"; +/** What a late answer is recorded as on a write that already settled unconfirmed. */ +export type WriteRecorded = "admitted_late" | "refused_late"; + +/** What the caller was told. Settled once; never changed by a later frame. */ +export type WriteSettlement = + | { outcome: "admitted" } + | { outcome: "refused"; code: string | null; error: string; response: CultNetErrorMessage | CultNetErrorV1Message } + | { outcome: "unconfirmed"; reason: WriteUnconfirmedReason }; + +/** A write's state: unanswered, or its settlement plus (after unconfirmed only) the late answer. */ +export type WriteState = + | { outcome: "pending"; recorded?: undefined } + | (WriteSettlement & { recorded?: WriteRecorded }); + +export interface WriteTrackerOptions { + /** The result window, from the send. Use `resultWindowMs(serverAdvertisesReceipts)`. */ + windowMs: number; + /** Called once per write, at the moment it settles. */ + onSettle?: (messageId: string, settlement: WriteSettlement) => void; +} + +interface Entry { + sentAtMs: number; + state: WriteState; +} + +export class WriteTracker { + readonly #windowMs: number; + readonly #onSettle?: (messageId: string, settlement: WriteSettlement) => void; + readonly #writes = new Map(); + + constructor(options: WriteTrackerOptions) { + this.#windowMs = options.windowMs; + this.#onSettle = options.onSettle; + } + + /** Registers a write sent at `atMs`. A messageId is minted once per session: re-sending a known id is the same write. */ + send(messageId: string, atMs: number): void { + this.tick(atMs); + if (!this.#writes.has(messageId)) this.#writes.set(messageId, { sentAtMs: atMs, state: { outcome: "pending" } }); + } + + /** Feeds one decoded frame received at `atMs`. Frames that are not answers to a write are ignored. */ + frame(message: CultNetMessage, atMs: number): void { + this.tick(atMs); + switch (message.schemaVersion) { + case "cultnet.write_admitted.v0": + this.#answer(message.messageId, { outcome: "admitted" }, "admitted_late"); + return; + case "cultnet.error.v1": + if (message.messageId !== null) { + this.#answer(message.messageId, refusalOf(message), "refused_late"); + return; + } + break; + case "cultnet.error.v0": + break; + default: + return; + } + // An error that names no write is attributable only when exactly one write is unanswered. + const unanswered = [...this.#writes].filter(([, entry]) => entry.state.outcome === "pending"); + if (unanswered.length === 1) this.#settle(unanswered[0][0], unanswered[0][1], refusalOf(message)); + } + + /** Advances the clock: every unanswered write whose window, counted from its send, has run settles unconfirmed. */ + tick(atMs: number): void { + for (const [messageId, entry] of this.#writes) { + if (entry.state.outcome === "pending" && atMs - entry.sentAtMs >= this.#windowMs) { + this.#settle(messageId, entry, { outcome: "unconfirmed", reason: "window_elapsed" }); + } + } + } + + /** The session ended: every unanswered write settles unconfirmed. */ + sessionEnd(atMs: number): void { + this.tick(atMs); + for (const [messageId, entry] of this.#writes) { + if (entry.state.outcome === "pending") this.#settle(messageId, entry, { outcome: "unconfirmed", reason: "session_ended" }); + } + } + + state(messageId: string): WriteState | undefined { + return this.#writes.get(messageId)?.state; + } + + /** Milliseconds from `atMs` until the next unanswered write's window runs out; undefined when none is unanswered. */ + nextDeadlineMs(atMs: number): number | undefined { + let next: number | undefined; + for (const entry of this.#writes.values()) { + if (entry.state.outcome !== "pending") continue; + const remaining = Math.max(0, entry.sentAtMs + this.#windowMs - atMs); + if (next === undefined || remaining < next) next = remaining; + } + return next; + } + + #answer(messageId: string, settlement: WriteSettlement, late: WriteRecorded): void { + const entry = this.#writes.get(messageId); + if (!entry) return; // names a write that is not in flight: settles nothing + if (entry.state.outcome === "pending") { + this.#settle(messageId, entry, settlement); + } else if (entry.state.outcome === "unconfirmed" && entry.state.recorded === undefined) { + entry.state = { ...entry.state, recorded: late }; + } + } + + #settle(messageId: string, entry: Entry, settlement: WriteSettlement): void { + entry.state = settlement; + this.#onSettle?.(messageId, settlement); + } +} + +function refusalOf(message: CultNetErrorMessage | CultNetErrorV1Message): WriteSettlement { + return { outcome: "refused", code: message.code, error: message.error, response: message }; +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts new file mode 100644 index 00000000..cc524bfd --- /dev/null +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -0,0 +1,181 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; + +import { + CULTNET_ERROR_CODES, + CULTNET_ERROR_V1_CODES, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + LEGACY_RESULT_WINDOW_MS, + RECEIPT_RESULT_WINDOW_MS, + WriteTracker, + cultNetSchemas, + helloAdvertisesWriteReceipts, + parseCultNetMessage, + resultWindowMs, + type CultNetErrorV1Message, + type CultNetMessage, + type CultNetWriteAdmittedMessage, + type WriteSettlement, +} from "../src"; + +const admitted = (messageId: string): CultNetWriteAdmittedMessage => + ({ schemaVersion: "cultnet.write_admitted.v0", messageId, operation: "put", schemaId: "note.v0", recordKey: "note:1" }); +const refusedV1 = (messageId: string | null, code: CultNetErrorV1Message["code"] = "write_not_authorized"): CultNetErrorV1Message => + ({ schemaVersion: "cultnet.error.v1", error: "no", routingHint: null, code, details: null, messageId }); +const errorV0 = (): CultNetMessage => + ({ schemaVersion: "cultnet.error.v0", error: "no", routingHint: null, code: "write_not_authorized", details: null }); + +test("error.v1 accepts every code of its closed enum, limit_reached included, and refuses any other", () => { + const schemaCodes = ((cultNetSchemas.errorV1Schema.properties.code as { enum: (string | null)[] }).enum).filter((c) => c !== null); + assert.deepEqual([...CULTNET_ERROR_V1_CODES].sort(), [...schemaCodes].sort()); + for (const code of CULTNET_ERROR_V1_CODES) parseCultNetMessage({ ...refusedV1("w1"), code }); + assert.throws(() => parseCultNetMessage({ ...refusedV1("w1"), code: "invented" })); +}); + +test("error.v1 echoes messageId byte for byte, allows nil, and requires the key", () => { + assert.equal((parseCultNetMessage(refusedV1("w-é 1")) as CultNetErrorV1Message).messageId, "w-é 1"); + assert.equal((parseCultNetMessage(refusedV1(null)) as CultNetErrorV1Message).messageId, null); + const { messageId: _omitted, ...withoutId } = refusedV1("w1"); + assert.throws(() => parseCultNetMessage(withoutId)); + assert.throws(() => parseCultNetMessage(refusedV1(""))); +}); + +test("error.v0 is unaffected: still closed, no messageId, no limit_reached", () => { + assert.deepEqual([...CULTNET_ERROR_CODES].includes("limit_reached" as never), false); + assert.throws(() => parseCultNetMessage({ ...errorV0(), code: "limit_reached" })); + assert.throws(() => parseCultNetMessage({ ...errorV0(), messageId: "w1" })); + assert.equal(parseCultNetMessage(errorV0()).schemaVersion, "cultnet.error.v0"); +}); + +test("write_admitted needs all five keys, a known operation and no error key", () => { + parseCultNetMessage(admitted("w1")); + parseCultNetMessage({ ...admitted("w1"), operation: "delete" }); + assert.throws(() => parseCultNetMessage({ ...admitted("w1"), operation: "upsert" })); + assert.throws(() => parseCultNetMessage({ ...admitted("w1"), error: "nope" })); + const { recordKey: _omitted, ...short } = admitted("w1"); + assert.throws(() => parseCultNetMessage(short)); +}); + +test("a hello is receipt-capable only when it lists both frames", () => { + assert.equal(helloAdvertisesWriteReceipts({ supportedMessageVersions: [...CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS] }), true); + assert.equal(helloAdvertisesWriteReceipts({ supportedMessageVersions: ["cultnet.write_admitted.v0"] }), false); + assert.equal(helloAdvertisesWriteReceipts({ supportedMessageVersions: ["cultnet.error.v1"] }), false); + assert.equal(helloAdvertisesWriteReceipts({}), false); + const hello = parseCultNetMessage({ schemaVersion: "cultnet.hello.v0", runtimeId: "r", runtimeKind: "k", supportedMessageVersions: [...CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS] }); + assert.equal(hello.schemaVersion, "cultnet.hello.v0"); +}); + +test("the window is 5000 ms for a receipt-capable server and 500 ms otherwise", () => { + assert.equal(resultWindowMs(true), RECEIPT_RESULT_WINDOW_MS); + assert.equal(resultWindowMs(false), LEGACY_RESULT_WINDOW_MS); + assert.equal(RECEIPT_RESULT_WINDOW_MS, 5000); + assert.equal(LEGACY_RESULT_WINDOW_MS, 500); +}); + +function tracker(windowMs = 500) { + const told: [string, WriteSettlement][] = []; + return { told, t: new WriteTracker({ windowMs, onSettle: (id, s) => told.push([id, s]) }) }; +} + +test("tracker: the window is inclusive at exactly windowMs and open one ms before", () => { + const { t, told } = tracker(); + t.send("w1", 100); + t.tick(599); + assert.equal(t.state("w1")?.outcome, "pending"); + t.tick(600); + assert.deepEqual(told, [["w1", { outcome: "unconfirmed", reason: "window_elapsed" }]]); +}); + +test("tracker: each write's window runs from its own send", () => { + const { t } = tracker(); + t.send("w1", 0); + t.send("w2", 300); + t.tick(500); + assert.equal(t.state("w1")?.outcome, "unconfirmed"); + assert.equal(t.state("w2")?.outcome, "pending"); +}); + +test("tracker: a frame arriving after the window cannot admit the write; it is recorded late", () => { + const { t, told } = tracker(); + t.send("w1", 0); + t.frame(admitted("w1"), 500); + assert.deepEqual(t.state("w1"), { outcome: "unconfirmed", reason: "window_elapsed", recorded: "admitted_late" }); + assert.equal(told.length, 1); +}); + +test("tracker: a write settles once; later frames neither change it nor flip a late record", () => { + const { t, told } = tracker(); + t.send("w1", 0); + t.frame(refusedV1("w1"), 10); + t.frame(admitted("w1"), 20); + t.frame(refusedV1("w1", "message_invalid"), 30); + t.sessionEnd(40); + assert.equal(told.length, 1); + assert.deepEqual(told[0][1], { outcome: "refused", code: "write_not_authorized", error: "no", response: refusedV1("w1") }); + t.send("w2", 100); + t.tick(600); + t.frame(refusedV1("w2"), 700); + t.frame(admitted("w2"), 800); + assert.equal(t.state("w2")?.outcome === "unconfirmed" && t.state("w2")?.recorded, "refused_late"); +}); + +test("tracker: an id-less error refuses the single unanswered write only, counting answered writes out", () => { + const { t } = tracker(); + t.send("w1", 0); + t.send("w2", 1); + t.frame(admitted("w1"), 5); + t.frame(refusedV1(null), 6); + assert.equal(t.state("w1")?.outcome, "admitted"); + assert.equal(t.state("w2")?.outcome, "refused"); +}); + +test("tracker: an id-less error never reaches a write that already ended unconfirmed", () => { + const { t } = tracker(); + t.send("w1", 0); + t.tick(500); + t.frame(errorV0(), 501); + assert.deepEqual(t.state("w1"), { outcome: "unconfirmed", reason: "window_elapsed" }); +}); + +test("tracker: sessionEnd settles unanswered writes only, and an expired window keeps its own reason", () => { + const { t } = tracker(); + t.send("w1", 0); + t.send("w2", 0); + t.send("w3", 400); + t.frame(admitted("w2"), 10); + t.sessionEnd(600); + assert.deepEqual(t.state("w1"), { outcome: "unconfirmed", reason: "window_elapsed" }); + assert.equal(t.state("w2")?.outcome, "admitted"); + assert.deepEqual(t.state("w3"), { outcome: "unconfirmed", reason: "session_ended" }); +}); + +test("tracker: re-sending a known id is the same write and does not restart its window", () => { + const { t } = tracker(); + t.send("w1", 0); + t.send("w1", 400); + t.tick(500); + assert.equal(t.state("w1")?.outcome, "unconfirmed"); +}); + +test("tracker: nextDeadlineMs reports the nearest unanswered window, none when all settled", () => { + const { t } = tracker(); + assert.equal(t.nextDeadlineMs(0), undefined); + t.send("w1", 0); + t.send("w2", 100); + assert.equal(t.nextDeadlineMs(50), 450); + assert.equal(t.nextDeadlineMs(900), 0); + t.frame(admitted("w1"), 60); + assert.equal(t.nextDeadlineMs(150), 450); + t.frame(admitted("w2"), 160); + assert.equal(t.nextDeadlineMs(170), undefined); +}); + +test("the replay runner passes every vector against this runtime", () => { + const script = join(__dirname, "..", "..", "tools", "replay-write-receipt-vectors.mjs"); + const run = spawnSync(process.execPath, [script], { encoding: "utf8" }); + assert.equal(run.status, 0, run.stdout + run.stderr); + assert.match(run.stdout, /^PASS scenario\/receipt_admits$/m); + assert.doesNotMatch(run.stdout, /^FAIL/m); +}); diff --git a/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs b/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs new file mode 100644 index 00000000..2c4bb403 --- /dev/null +++ b/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node +// Replays contracts/cultnet/interop/write-receipt-vectors.json against this runtime: every `scenarios.cases` event +// replay through a fake server and the WriteTracker, and every `wire` / `schemaInvalid` frame through the real +// decoder. Prints PASS/FAIL per case; exit 1 on any failure. Runs against the built package (`npm run build` first). +// +// node tools/replay-write-receipt-vectors.mjs [path/to/write-receipt-vectors.json] +// +// The runner interface a port (Rust, C#) must reproduce is documented in README.md, "Write receipt replay runner". + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const here = path.dirname(fileURLToPath(import.meta.url)); +const { decode, encode } = require("@msgpack/msgpack"); +const { + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, LEGACY_RESULT_WINDOW_MS, RECEIPT_RESULT_WINDOW_MS, WriteTracker, + encodeCultNetMessageForWire, helloAdvertisesWriteReceipts, parseCultNetMessage, resultWindowMs, +} = require(path.join(here, "..", "dist", "index.js")); + +const vectorsPath = process.argv[2] ?? path.join(here, "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); +const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); + +const results = []; +const check = (name, run) => { + try { + const problems = run(); + results.push({ name, problems }); + } catch (error) { + results.push({ name, problems: [`threw: ${error instanceof Error ? error.message : String(error)}`] }); + } +}; +const bytesOf = (hex) => Uint8Array.from(Buffer.from(hex, "hex")); +const hexOf = (bytes) => Buffer.from(bytes).toString("hex").toUpperCase(); +// A frame reaches the client as bytes: the fake server encodes it, the client decodes and validates it. +const delivered = (frame) => parseCultNetMessage(decode(encode(frame))); + +check("constants", () => [ + ...(vectors.constants.RECEIPT_RESULT_WINDOW_MS === RECEIPT_RESULT_WINDOW_MS ? [] : [`RECEIPT window ${RECEIPT_RESULT_WINDOW_MS} != ${vectors.constants.RECEIPT_RESULT_WINDOW_MS}`]), + ...(vectors.constants.LEGACY_RESULT_WINDOW_MS === LEGACY_RESULT_WINDOW_MS ? [] : [`LEGACY window ${LEGACY_RESULT_WINDOW_MS} != ${vectors.constants.LEGACY_RESULT_WINDOW_MS}`]), +]); + +for (const vector of vectors.wire) { + check(`wire/${vector.name}`, () => { + const problems = []; + const parsed = parseCultNetMessage(decode(bytesOf(vector.messagePackHex))); + if (JSON.stringify(sorted(parsed)) !== JSON.stringify(sorted(vector.message))) problems.push(`decoded ${JSON.stringify(parsed)} != ${JSON.stringify(vector.message)}`); + const reencoded = hexOf(encode(encodeCultNetMessageForWire(parsed))); + if (reencoded !== vector.messagePackHex.toUpperCase()) problems.push(`re-encoded bytes ${reencoded} != captured ${vector.messagePackHex}`); + return problems; + }); +} + +for (const vector of vectors.schemaInvalid) { + check(`schemaInvalid/${vector.name}`, () => { + try { + parseCultNetMessage(decode(bytesOf(vector.messagePackHex))); + } catch { + return []; + } + return [`accepted a frame that must be refused (${vector.why})`]; + }); +} + +for (const scenario of vectors.scenarios.cases) { + check(`scenario/${scenario.name}`, () => { + // The fake server states its capability the only way a real one does: in its hello. + const serverHello = { schemaVersion: "cultnet.hello.v0", runtimeId: "fake-server", runtimeKind: "fake", + ...(scenario.server === "receipt" ? { supportedMessageVersions: [...CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS] } : {}) }; + const windowMs = resultWindowMs(helloAdvertisesWriteReceipts(serverHello)); + const problems = []; + if (windowMs !== vectors.constants[scenario.window]) problems.push(`window ${windowMs} != ${scenario.window}`); + + const told = new Map(); // what the caller was told, at settle time + const tracker = new WriteTracker({ windowMs, onSettle: (id, settlement) => { + if (told.has(id)) problems.push(`${id} settled twice`); + told.set(id, settlement); + } }); + for (const event of [...scenario.events].sort((a, b) => a.atMs - b.atMs)) { + if ("send" in event) tracker.send(event.send, event.atMs); + else if ("frame" in event) tracker.frame(delivered(event.frame), event.atMs); + else if ("tick" in event) tracker.tick(event.atMs); + else if ("sessionEnd" in event) tracker.sessionEnd(event.atMs); + // transportAck: delivery is not an answer; the tracker has no such input and must not need one. + } + for (const [id, want] of Object.entries(scenario.expect)) { + const got = told.get(id); + if (!got) { problems.push(`${id}: never settled, wanted ${want.outcome}`); continue; } + if (got.outcome !== want.outcome) problems.push(`${id}: told ${got.outcome}, wanted ${want.outcome}`); + if (want.code !== undefined && got.code !== want.code) problems.push(`${id}: code ${got.code}, wanted ${want.code}`); + if (want.reason !== undefined && got.reason !== want.reason) problems.push(`${id}: reason ${got.reason}, wanted ${want.reason}`); + const recorded = tracker.state(id)?.recorded; + if (recorded !== want.recorded) problems.push(`${id}: recorded ${recorded}, wanted ${want.recorded}`); + } + for (const id of told.keys()) if (!(id in scenario.expect)) problems.push(`${id}: settled but the scenario expects nothing for it`); + return problems; + }); +} + +function sorted(value) { + if (Array.isArray(value)) return value.map(sorted); + if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((k) => [k, sorted(value[k])])); + return value; +} + +let failed = 0; +for (const { name, problems } of results) { + if (problems.length === 0) { console.log(`PASS ${name}`); continue; } + failed++; + console.log(`FAIL ${name}`); + for (const problem of problems) console.log(` ${problem}`); +} +console.log(`${results.length - failed}/${results.length} passed`); +process.exit(failed === 0 ? 0 : 1); From a67faad27c912130dc355afb8c221bbc9b5cc5a1 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 15:04:57 +0000 Subject: [PATCH 056/117] cultnet-ts: tests for two tracker mutants (send expires other windows; non-answer frames settle nothing) Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/test/write-receipt.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index cc524bfd..66b80601 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -159,6 +159,21 @@ test("tracker: re-sending a known id is the same write and does not restart its assert.equal(t.state("w1")?.outcome, "unconfirmed"); }); +test("tracker: a send past another write's window settles that write at once", () => { + const { t, told } = tracker(); + t.send("w1", 0); + t.send("w2", 600); + assert.deepEqual(told, [["w1", { outcome: "unconfirmed", reason: "window_elapsed" }]]); +}); + +test("tracker: only receipt and error frames answer; any other frame naming the id settles nothing", () => { + const { t } = tracker(); + t.send("w1", 0); + t.frame({ schemaVersion: "cultnet.hello.v0", runtimeId: "r", runtimeKind: "k" }, 1); + t.frame({ schemaVersion: "cultnet.document_put_raw.v0", messageId: "w1" } as unknown as CultNetMessage, 2); + assert.equal(t.state("w1")?.outcome, "pending"); +}); + test("tracker: nextDeadlineMs reports the nearest unanswered window, none when all settled", () => { const { t } = tracker(); assert.equal(t.nextDeadlineMs(0), undefined); From 293b4e83bc3e379a0f52c8bfcd8025d374851b03 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 16:40:52 +0000 Subject: [PATCH 057/117] cultmesh-ts, cultnet-ts: an unanswered put is unconfirmed on every server, session_ended before a result is an error, a failed local send is its own reason Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/CHANGELOG.md | 2 +- packages/cultmesh-ts/src/index.ts | 44 +++++++++--------- .../test/client-write-completion.test.ts | 11 +++-- packages/cultmesh-ts/test/cultmesh.test.ts | 45 ++++++++++--------- .../test/write-receipt-client.test.ts | 35 ++++++++++++--- packages/cultnet-ts/src/write-tracker.ts | 13 +++++- .../cultnet-ts/test/write-receipt.test.ts | 26 +++++++++++ 7 files changed, 124 insertions(+), 52 deletions(-) diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index f803476a..e7b432a5 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following ### Added -- `publishRudpDocumentOnce` takes `writeReceipts: true`: it sends a hello listing `cultnet.write_admitted.v0` and `cultnet.error.v1` and, against a server that lists both back, resolves only on that write's `write_admitted`, rejects with `CultMeshRudpWriteRefusedError` (now also carrying an `error.v1`) on its refusal and with the new `CultMeshRudpWriteUnconfirmedError` (`reason`: `window_elapsed` after 5000 ms, or `session_ended`) when nothing settles it. Without the option, or against a server that did not advertise, nothing changes: no hello, a 500 ms window, and a window with no refusal still reads as published (interim; not admission). `resultWindowMs` overrides either window. `publishRudpDocumentAndWaitForReceipt` is not changed. +- `publishRudpDocumentOnce` takes `writeReceipts: true`: it sends a hello listing `cultnet.write_admitted.v0` and `cultnet.error.v1` and, against a server that lists both back, resolves only on that write's `write_admitted`, rejects with `CultMeshRudpWriteRefusedError` (now also carrying an `error.v1`) on its refusal and with the new `CultMeshRudpWriteUnconfirmedError` (`reason`: `window_elapsed` after 5000 ms, `session_ended`, or `local_send_failed`) when nothing settles it. Without the option, or against a server that did not advertise, there is no hello and the window is 500 ms; a put nothing settled in it is unconfirmed too, no longer "published" (**behaviour change**: a server that applies a put but answers nothing now yields `CultMeshRudpWriteUnconfirmedError`, as the spec requires). The legacy success reply (the put's id echoed) still admits it. `resultWindowMs` overrides either window. `publishRudpDocumentAndWaitForReceipt` is not changed. ### Breaking diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index eab2e6dc..43a44c9b 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4205,8 +4205,8 @@ export class CultMeshRudpWriteRefusedError extends Error { } /** - * Nothing settled the write (write-receipt.md, "unconfirmed"): the result window ran out with no answer, or the - * session ended first. Not success and not refusal; a caller that needs admission treats it as not admitted. + * Nothing settled the write (write-receipt.md, "unconfirmed"): the result window ran out with no answer, the + * session ended first, or the client's own send failed. Not success and not refusal; a caller that needs admission treats it as not admitted. */ export class CultMeshRudpWriteUnconfirmedError extends Error { readonly reason: WriteUnconfirmedReason; @@ -4223,9 +4223,9 @@ export interface CultMeshRudpDocumentPublishOptions extends CultMeshRudpPeerOpti /** * How long, counted from the send, the put may wait for its result frame. A reliable-transport ack is not * admission; the outcome is the peer's result frame. Default: 5000 ms against a server whose hello lists the - * write-receipt frames, 500 ms otherwise. Against a server that did not advertise them the put is reported - * published once the window passes with no refusal (interim; the caller must not read it as admission), and - * a success reply ends the wait early. + * write-receipt frames, 500 ms otherwise. A put nothing settled within the window is unconfirmed, never + * published, whatever the server advertised; a legacy server's success reply (the put's id echoed back) ends + * the wait early as admitted. */ resultWindowMs?: number; /** @@ -6250,19 +6250,24 @@ export class CultMesh { }); tracker = writeTracker; writeTracker.send(messageId, Date.now()); - peer.send(registry.createRawDocumentPutMessage( - binding, - messageId, - recordKey, - value, - { - sourceRuntimeId: options.sourceRuntimeId ?? runtimeId, - sourceAgentId: options.sourceAgentId, - sourceRole: options.sourceRole, - tags: options.tags, - }, - )); - await peer.flush(options.flushTimeoutMs ?? 30_000); + try { + peer.send(registry.createRawDocumentPutMessage( + binding, + messageId, + recordKey, + value, + { + sourceRuntimeId: options.sourceRuntimeId ?? runtimeId, + sourceAgentId: options.sourceAgentId, + sourceRole: options.sourceRole, + tags: options.tags, + }, + )); + await peer.flush(options.flushTimeoutMs ?? 30_000); + } catch { + // Our own send failed or was never acknowledged: the put may or may not have left. Never a refusal. + writeTracker.sendFailed(messageId, Date.now()); + } while (!settlement) { const remaining = writeTracker.nextDeadlineMs(Date.now()) ?? 0; await new Promise((resolve) => { @@ -6272,8 +6277,7 @@ export class CultMesh { writeTracker.tick(Date.now()); } if (settlement.outcome === "refused") throw new CultMeshRudpWriteRefusedError(settlement.response); - // Interim, for a server that did not advertise receipts: a window with no refusal reads as published. - if (settlement.outcome === "unconfirmed" && receiptCapable) throw new CultMeshRudpWriteUnconfirmedError(messageId, settlement.reason); + if (settlement.outcome === "unconfirmed") throw new CultMeshRudpWriteUnconfirmedError(messageId, settlement.reason); } finally { peer.close(); } diff --git a/packages/cultmesh-ts/test/client-write-completion.test.ts b/packages/cultmesh-ts/test/client-write-completion.test.ts index 8dd122ed..5c4a1eaa 100644 --- a/packages/cultmesh-ts/test/client-write-completion.test.ts +++ b/packages/cultmesh-ts/test/client-write-completion.test.ts @@ -16,7 +16,7 @@ import { encodeRudpPacket, } from "@gamecult/cultnet-ts"; import { encodeWireValue } from "@gamecult/cultnet-ts/wire"; -import { CultMesh, CultMeshGrantAllWrites, CultMeshRudpWriteRefusedError } from "../src/index"; +import { CultMesh, CultMeshGrantAllWrites, CultMeshRudpWriteRefusedError, CultMeshRudpWriteUnconfirmedError } from "../src/index"; const isTheRefusal = (error: unknown): boolean => error instanceof CultMeshRudpWriteRefusedError && error.code === "write_not_authorized"; @@ -100,7 +100,7 @@ for (const order of orders) { }); } -test("publishRudpDocumentOnce still publishes a put the server admitted", async () => { +test("a put the server applied but did not answer is unconfirmed, never published", async () => { const connectionId = nextConnection++; const puts: string[] = []; const server = CultMesh.createRudpDocumentServer("completion-admit", connectionId, { @@ -111,7 +111,10 @@ test("publishRudpDocumentOnce still publishes a put the server admitted", async }); await server.start(); try { - await CultMesh.publishRudpDocumentOnce("completion-admit-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, binding, "tests:completion:a", { noteId: "a", text: "x" }, { resendDelayMs: 50, resendPollMs: 5, resultWindowMs: 200 }); - assert.deepEqual(puts, ["tests:completion:a"]); + await assert.rejects( + CultMesh.publishRudpDocumentOnce("completion-admit-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, binding, "tests:completion:a", { noteId: "a", text: "x" }, { resendDelayMs: 50, resendPollMs: 5, resultWindowMs: 200 }), + (error: unknown) => error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed", + ); + assert.deepEqual(puts, ["tests:completion:a"], "the server did apply it; this server just does not answer"); } finally { server.close(); } }); diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 90ff38f9..68429969 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -21,7 +21,7 @@ import { type CultNetMessage, type CultNetSnapshotResponseRawMessage, } from "@gamecult/cultnet-ts"; -import { CultMesh, CultMeshGrantAllWrites, CultMeshRudpUnservableDocumentError } from "../src/index"; +import { CultMesh, CultMeshGrantAllWrites, CultMeshRudpUnservableDocumentError, CultMeshRudpWriteUnconfirmedError } from "../src/index"; // A record's payload as the bytes a peer would put on the wire. const payloadOf = (value: unknown) => documentPayloadFromBytes(encodeWireValue(value)); @@ -2968,25 +2968,30 @@ test("CultMesh TS publishes one registered document to a RUDP catalog", async () try { await server.start(); - await CultMesh.publishRudpDocumentOnce( - "cultmesh-ts-muninn", - connectionId, - `rudp://127.0.0.1:${server.bind.port}`, - defineCultNetDocumentBinding({ definition: noteDocument }), - "note:odin", - { - noteId: "note:odin", - body: "pay respects once", - }, - { - resendDelayMs: 25, - resendPollMs: 5, - maxFragmentBytes: 1024, - maxPendingReliablePackets: 16, - connectTimeoutMs: 1_000, - flushTimeoutMs: 25, - sourceRole: "test-provider", - }, + // This server answers a put with nothing, so the writer cannot know the put landed: unconfirmed, though it did. + await assert.rejects( + CultMesh.publishRudpDocumentOnce( + "cultmesh-ts-muninn", + connectionId, + `rudp://127.0.0.1:${server.bind.port}`, + defineCultNetDocumentBinding({ definition: noteDocument }), + "note:odin", + { + noteId: "note:odin", + body: "pay respects once", + }, + { + resendDelayMs: 25, + resendPollMs: 5, + maxFragmentBytes: 1024, + maxPendingReliablePackets: 16, + connectTimeoutMs: 1_000, + flushTimeoutMs: 25, + resultWindowMs: 100, + sourceRole: "test-provider", + }, + ), + (error: unknown) => error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed", ); await waitFor(() => received !== undefined, "published RUDP document"); diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index ddc3d41e..e6da5bf2 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -31,8 +31,10 @@ interface Script { /** Does the server's hello list both receipt frames? */ advertise: boolean; /** What the server answers a put with, `delayMs` after it arrives. */ - answer: "admit" | "refuse" | "wrong-id" | "silence"; + answer: "admit" | "refuse" | "wrong-id" | "silence" | "goodbye"; delayMs?: number; + /** Acknowledge the reliable packets it receives? Default true. */ + ack?: boolean; } async function fakeServer(connectionId: number, script: Script) { @@ -72,7 +74,9 @@ async function fakeServer(connectionId: number, script: Script) { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || script.answer === "silence") continue; const timer = setTimeout(() => { timers.delete(timer); - if (script.answer === "refuse") { + if (script.answer === "goodbye") { + send(session.endUnsendableSession(new Error("scripted goodbye")), remote.port, remote.address); + } else if (script.answer === "refuse") { say({ schemaVersion: "cultnet.error.v1", error: "write_not_authorized", routingHint: null, code: "write_not_authorized", details: null, messageId: message.messageId! }, remote.port, remote.address); } else { const messageId = script.answer === "wrong-id" ? "someone-else" : message.messageId!; @@ -81,7 +85,7 @@ async function fakeServer(connectionId: number, script: Script) { }, script.delayMs ?? 20); timers.add(timer); } - if (packet.reliable) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); + if (packet.reliable && script.ack !== false) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); }); return { endpoint: `rudp://127.0.0.1:${socket.address().port}`, seen, close: () => { closed = true; timers.forEach(clearTimeout); socket.close(); } }; } @@ -121,19 +125,38 @@ test("a receipt for another id settles nothing", async () => { } finally { server.close(); } }); -test("a server whose hello does not advertise keeps the interim 500 ms window-success", async () => { +test("a server whose hello does not advertise leaves a silent put unconfirmed after the 500 ms window, never published", async () => { const server = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); const started = Date.now(); try { - await publish(server, { writeReceipts: true }); + await assert.rejects(publish(server, { writeReceipts: true }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed"); assert.ok(Date.now() - started >= 500, "the legacy window ran from the send"); } finally { server.close(); } }); +test("a session that ends before any result leaves the put unconfirmed (session_ended), against a legacy server too", async () => { + for (const advertise of [false, true]) { + const server = await fakeServer(nextConnection, { advertise, answer: "goodbye" }); + try { + await assert.rejects(publish(server, { writeReceipts: true, resultWindowMs: 3_000 }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "session_ended"); + } finally { server.close(); } + } +}); + +test("a put whose send is never acknowledged is unconfirmed as local_send_failed, not an untyped error", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "silence", ack: false }); + try { + await assert.rejects(publish(server, { writeReceipts: true, flushTimeoutMs: 300 }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "local_send_failed"); + } finally { server.close(); } +}); + test("without writeReceipts no hello is sent and the receipt frames are never asked for", async () => { const server = await fakeServer(nextConnection, { advertise: true, answer: "silence" }); try { - await publish(server, { resultWindowMs: 100 }); + await assert.rejects(publish(server, { resultWindowMs: 100 }), CultMeshRudpWriteUnconfirmedError); assert.deepEqual(server.seen, ["cultnet.document_put_raw.v0"]); } finally { server.close(); } }); diff --git a/packages/cultnet-ts/src/write-tracker.ts b/packages/cultnet-ts/src/write-tracker.ts index 7398afaa..11229232 100644 --- a/packages/cultnet-ts/src/write-tracker.ts +++ b/packages/cultnet-ts/src/write-tracker.ts @@ -14,7 +14,11 @@ export function resultWindowMs(serverAdvertisesReceipts: boolean): number { return serverAdvertisesReceipts ? RECEIPT_RESULT_WINDOW_MS : LEGACY_RESULT_WINDOW_MS; } -export type WriteUnconfirmedReason = "window_elapsed" | "session_ended"; +/** + * Why nothing settled a write: its window ran out, the session ended first, or the client's own send of it failed (it may or + * may not have left the machine, and is never a refusal). + */ +export type WriteUnconfirmedReason = "window_elapsed" | "session_ended" | "local_send_failed"; /** What a late answer is recorded as on a write that already settled unconfirmed. */ export type WriteRecorded = "admitted_late" | "refused_late"; @@ -97,6 +101,13 @@ export class WriteTracker { } } + /** This client's own send of `messageId` failed: that write settles unconfirmed; the others are untouched. */ + sendFailed(messageId: string, atMs: number): void { + this.tick(atMs); + const entry = this.#writes.get(messageId); + if (entry?.state.outcome === "pending") this.#settle(messageId, entry, { outcome: "unconfirmed", reason: "local_send_failed" }); + } + state(messageId: string): WriteState | undefined { return this.#writes.get(messageId)?.state; } diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 66b80601..31a1c388 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -194,3 +194,29 @@ test("the replay runner passes every vector against this runtime", () => { assert.match(run.stdout, /^PASS scenario\/receipt_admits$/m); assert.doesNotMatch(run.stdout, /^FAIL/m); }); + +test("a failed local send settles only that write unconfirmed as local_send_failed, and an answer after it is recorded late", () => { + const settled: [string, WriteSettlement][] = []; + const tracker = new WriteTracker({ windowMs: 1000, onSettle: (id, settlement) => settled.push([id, settlement]) }); + tracker.send("w1", 0); + tracker.send("w2", 0); + tracker.sendFailed("w1", 10); + assert.deepEqual(settled, [["w1", { outcome: "unconfirmed", reason: "local_send_failed" }]]); + assert.equal(tracker.state("w2")?.outcome, "pending"); + tracker.frame(admitted("w1"), 20); + assert.deepEqual(tracker.state("w1"), { outcome: "unconfirmed", reason: "local_send_failed", recorded: "admitted_late" }); + tracker.sendFailed("w2", 30); + tracker.sendFailed("w2", 31); + assert.equal(settled.length, 2, "a write settles once"); + tracker.sendFailed("unknown", 40); + assert.equal(tracker.state("unknown"), undefined); +}); + +test("a failed local send advances the clock: other writes whose windows have run settle window_elapsed", () => { + const tracker = new WriteTracker({ windowMs: 1000 }); + tracker.send("w1", 0); + tracker.send("w2", 500); + tracker.sendFailed("w2", 1200); + assert.deepEqual(tracker.state("w1"), { outcome: "unconfirmed", reason: "window_elapsed" }); + assert.deepEqual(tracker.state("w2"), { outcome: "unconfirmed", reason: "local_send_failed" }); +}); From af1ac72359187291489154f156b3a9c5bd7580c9 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 17:05:42 +0000 Subject: [PATCH 058/117] cultmesh-ts, cultnet-ts: only write_admitted admits a put (no echo admission); the send is bounded by the result window and a timeout is local_send_failed Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/CHANGELOG.md | 2 +- packages/cultmesh-ts/src/index.ts | 13 +++++----- .../test/write-receipt-client.test.ts | 24 +++++++++++++++++-- packages/cultnet-ts/src/write-tracker.ts | 2 +- .../cultnet-ts/test/write-receipt.test.ts | 7 ++++++ 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index e7b432a5..f4612f2e 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following ### Added -- `publishRudpDocumentOnce` takes `writeReceipts: true`: it sends a hello listing `cultnet.write_admitted.v0` and `cultnet.error.v1` and, against a server that lists both back, resolves only on that write's `write_admitted`, rejects with `CultMeshRudpWriteRefusedError` (now also carrying an `error.v1`) on its refusal and with the new `CultMeshRudpWriteUnconfirmedError` (`reason`: `window_elapsed` after 5000 ms, `session_ended`, or `local_send_failed`) when nothing settles it. Without the option, or against a server that did not advertise, there is no hello and the window is 500 ms; a put nothing settled in it is unconfirmed too, no longer "published" (**behaviour change**: a server that applies a put but answers nothing now yields `CultMeshRudpWriteUnconfirmedError`, as the spec requires). The legacy success reply (the put's id echoed) still admits it. `resultWindowMs` overrides either window. `publishRudpDocumentAndWaitForReceipt` is not changed. +- `publishRudpDocumentOnce` takes `writeReceipts: true`: it sends a hello listing `cultnet.write_admitted.v0` and `cultnet.error.v1` and, against a server that lists both back, resolves only on that write's `write_admitted`, rejects with `CultMeshRudpWriteRefusedError` (now also carrying an `error.v1`) on its refusal and with the new `CultMeshRudpWriteUnconfirmedError` (`reason`: `window_elapsed` after 5000 ms, `session_ended`, or `local_send_failed`) when nothing settles it. Without the option, or against a server that did not advertise, there is no hello and the window is 500 ms; a put nothing settled in it is unconfirmed too, no longer "published" (**behaviour change**: a server that applies a put but answers nothing now yields `CultMeshRudpWriteUnconfirmedError`, as the spec requires). A legacy server's echo of the put's id is not admission: only `write_admitted` is. The flush that sends the put is bounded by the result window, and a send not acknowledged by then is `local_send_failed`. `resultWindowMs` overrides either window. `publishRudpDocumentAndWaitForReceipt` is not changed. ### Breaking diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 43a44c9b..be053f53 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4224,8 +4224,9 @@ export interface CultMeshRudpDocumentPublishOptions extends CultMeshRudpPeerOpti * How long, counted from the send, the put may wait for its result frame. A reliable-transport ack is not * admission; the outcome is the peer's result frame. Default: 5000 ms against a server whose hello lists the * write-receipt frames, 500 ms otherwise. A put nothing settled within the window is unconfirmed, never - * published, whatever the server advertised; a legacy server's success reply (the put's id echoed back) ends - * the wait early as admitted. + * published, whatever the server advertised: only a `write_admitted` is admission. The send itself is also + * bounded by the window (`flushTimeoutMs` is the upper bound); a send not acknowledged by then is + * unconfirmed as `local_send_failed`. */ resultWindowMs?: number; /** @@ -6225,10 +6226,6 @@ export class CultMesh { if (message.schemaVersion === "cultnet.hello.v0") { receiptCapable = helloAdvertisesWriteReceipts(message); serverHello?.(); - } else if (!receiptCapable && "messageId" in message && message.messageId === messageId - && message.schemaVersion !== "cultnet.error.v1") { - // A server that did not advertise receipts may echo the id back as its success reply. - tracker?.frame({ schemaVersion: "cultnet.write_admitted.v0", messageId, operation: "put", schemaId: binding.definition.schemaId ?? "", recordKey }, Date.now()); } else { tracker?.frame(message, Date.now()); } @@ -6263,7 +6260,9 @@ export class CultMesh { tags: options.tags, }, )); - await peer.flush(options.flushTimeoutMs ?? 30_000); + // The flush may not outlast the result window: a send nobody acknowledged by then is a local send failure. + const windowRemaining = writeTracker.nextDeadlineMs(Date.now()) ?? 0; + await peer.flush(Math.min(options.flushTimeoutMs ?? 30_000, Math.max(windowRemaining, 1))); } catch { // Our own send failed or was never acknowledged: the put may or may not have left. Never a refusal. writeTracker.sendFailed(messageId, Date.now()); diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index e6da5bf2..cf07138b 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -31,7 +31,7 @@ interface Script { /** Does the server's hello list both receipt frames? */ advertise: boolean; /** What the server answers a put with, `delayMs` after it arrives. */ - answer: "admit" | "refuse" | "wrong-id" | "silence" | "goodbye"; + answer: "admit" | "refuse" | "wrong-id" | "silence" | "goodbye" | "echo"; delayMs?: number; /** Acknowledge the reliable packets it receives? Default true. */ ack?: boolean; @@ -74,7 +74,9 @@ async function fakeServer(connectionId: number, script: Script) { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || script.answer === "silence") continue; const timer = setTimeout(() => { timers.delete(timer); - if (script.answer === "goodbye") { + if (script.answer === "echo") { + say({ schemaVersion: "cultnet.document_put_raw.v0", messageId: message.messageId!, document: { schemaId: "cultmesh.receipt_note.v0", recordKey: "tests:receipt:a", storedAt: "t", payloadEncoding: "messagepack", payload: new Uint8Array([1]) } } as unknown as CultNetMessage, remote.port, remote.address); + } else if (script.answer === "goodbye") { send(session.endUnsendableSession(new Error("scripted goodbye")), remote.port, remote.address); } else if (script.answer === "refuse") { say({ schemaVersion: "cultnet.error.v1", error: "write_not_authorized", routingHint: null, code: "write_not_authorized", details: null, messageId: message.messageId! }, remote.port, remote.address); @@ -153,6 +155,24 @@ test("a put whose send is never acknowledged is unconfirmed as local_send_failed } finally { server.close(); } }); +test("the flush is bounded by the result window: a send nobody acknowledges is local_send_failed at the window, not at the flush timeout", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "silence", ack: false }); + const started = Date.now(); + try { + await assert.rejects(publish(server, { writeReceipts: true, resultWindowMs: 300, flushTimeoutMs: 20_000 }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "local_send_failed"); + assert.ok(Date.now() - started < 3_000, "it did not wait out the 20 s flush timeout"); + } finally { server.close(); } +}); + +test("a server that does not advertise receipts and echoes the put's id back has not admitted it", async () => { + const server = await fakeServer(nextConnection, { advertise: false, answer: "echo" }); + try { + await assert.rejects(publish(server, { writeReceipts: true, resultWindowMs: 300 }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed"); + } finally { server.close(); } +}); + test("without writeReceipts no hello is sent and the receipt frames are never asked for", async () => { const server = await fakeServer(nextConnection, { advertise: true, answer: "silence" }); try { diff --git a/packages/cultnet-ts/src/write-tracker.ts b/packages/cultnet-ts/src/write-tracker.ts index 11229232..2204d371 100644 --- a/packages/cultnet-ts/src/write-tracker.ts +++ b/packages/cultnet-ts/src/write-tracker.ts @@ -103,9 +103,9 @@ export class WriteTracker { /** This client's own send of `messageId` failed: that write settles unconfirmed; the others are untouched. */ sendFailed(messageId: string, atMs: number): void { - this.tick(atMs); const entry = this.#writes.get(messageId); if (entry?.state.outcome === "pending") this.#settle(messageId, entry, { outcome: "unconfirmed", reason: "local_send_failed" }); + this.tick(atMs); } state(messageId: string): WriteState | undefined { diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 31a1c388..83c31411 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -220,3 +220,10 @@ test("a failed local send advances the clock: other writes whose windows have ru assert.deepEqual(tracker.state("w1"), { outcome: "unconfirmed", reason: "window_elapsed" }); assert.deepEqual(tracker.state("w2"), { outcome: "unconfirmed", reason: "local_send_failed" }); }); + +test("a write whose own send failed settles local_send_failed even when its window has also run", () => { + const tracker = new WriteTracker({ windowMs: 1000 }); + tracker.send("w1", 0); + tracker.sendFailed("w1", 1000); + assert.deepEqual(tracker.state("w1"), { outcome: "unconfirmed", reason: "local_send_failed" }); +}); From c89c4ee39ab552b2a44a3093600a33c3164d89a0 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 17:07:54 +0000 Subject: [PATCH 059/117] cultmesh-ts: test kills the flush-cap mutant Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/test/write-receipt-client.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index cf07138b..19f27a3c 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -149,9 +149,11 @@ test("a session that ends before any result leaves the put unconfirmed (session_ test("a put whose send is never acknowledged is unconfirmed as local_send_failed, not an untyped error", async () => { const server = await fakeServer(nextConnection, { advertise: true, answer: "silence", ack: false }); + const started = Date.now(); try { await assert.rejects(publish(server, { writeReceipts: true, flushTimeoutMs: 300 }), (error: unknown) => error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "local_send_failed"); + assert.ok(Date.now() - started < 2_500, "flushTimeoutMs bounds the send inside the 5 s window"); } finally { server.close(); } }); From 0caef4d564e695033b179d171d6bf1b10a25c5d4 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 17:38:47 +0000 Subject: [PATCH 060/117] WIP (unverified): publishRudpDocumentOnce sendOnly + presence publisher send-only; tests written, cultmesh-ts receipt-client run hung Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/CHANGELOG.md | 1 + packages/cultmesh-ts/src/index.ts | 46 ++++++++--- .../test/write-receipt-client.test.ts | 31 +++++++ .../src/idunn-odin-presence-publisher.ts | 30 ++++--- .../idunn-odin-presence-send-only.test.ts | 82 +++++++++++++++++++ 5 files changed, 164 insertions(+), 26 deletions(-) create mode 100644 packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts diff --git a/packages/cultmesh-ts/CHANGELOG.md b/packages/cultmesh-ts/CHANGELOG.md index f4612f2e..5df85ad3 100644 --- a/packages/cultmesh-ts/CHANGELOG.md +++ b/packages/cultmesh-ts/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following ### Breaking +- `publishRudpDocumentOnce` no longer treats a window with no refusal as published: a put nothing settled in its result window (500 ms, or 5000 ms against a receipt-capable server) rejects `CultMeshRudpWriteUnconfirmedError`, and a legacy server's echo of the put's id is not admission. It now resolves `"admitted"` (only on a `write_admitted`) or `"sent"`. **Migration:** a caller whose server cannot answer a write yet (for example Odin, until it is receipt-capable) passes `sendOnly: true` per call and treats `"sent"` as "sent", never as admitted. Receipts stay opt-in (`writeReceipts: true`); neither mode is the default. - The package is renamed `cultmesh-ts` to `@gamecult/cultmesh-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-ts` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`. - `asSchemaAlias` is removed from `CultMeshDocumentHandle`, `CultMeshCollectionHandle` and their bound forms. Read the handle by the schema you mean. - `CultMeshDocumentCatalog` (`document`, `tryDocument`, `latest`, `replace`, `submitPrediction`, `authoritativeWriter`, `predictionWriter`, `watch`) and `CultMeshCollectionCatalog` (`collection`, `tryCollection`, `latest`, `watchChanges`) take `` and yield `CultMeshCatalogValue`: an explicit type argument wins, a CultCache document definition names its own value type, and a bare schema descriptor yields `unknown`. A value type is no longer inferred from a value or an expected result. The `parse` option is removed from these calls, and a lookup by type answers with that type's handle or none, not an alias of a compatible schema. diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index be053f53..38c0b34a 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4239,6 +4239,14 @@ export interface CultMeshRudpDocumentPublishOptions extends CultMeshRudpPeerOpti writeReceipts?: boolean; /** With `writeReceipts`, how long to wait for the server's hello before treating it as legacy. Default 500 ms. */ helloWaitMs?: number; + /** + * Send the put and report only that it was sent: no hello, no wait for a result frame. Resolves `"sent"` once the + * transport has acknowledged the put (`flushTimeoutMs`), rejects `CultMeshRudpWriteUnconfirmedError` + * (`local_send_failed`) if it cannot be sent, and never resolves `"admitted"`: whether the server applied or + * refused it is unknown. For callers whose server cannot answer a write yet. Opt-in per call; not combinable with + * `writeReceipts`. + */ + sendOnly?: boolean; messageId?: string; sourceRuntimeId?: string; sourceAgentId?: string; @@ -6208,13 +6216,35 @@ export class CultMesh { recordKey: string, value: CultCacheDocumentValue, options: CultMeshRudpDocumentPublishOptions = {}, - ): Promise { + ): Promise<"admitted" | "sent"> { requireNonEmpty(runtimeId, "runtimeId"); requireNonEmpty(recordKey, "recordKey"); + if (options.sendOnly && options.writeReceipts) throw new TypeError("sendOnly and writeReceipts cannot be combined."); const registry = new CultNetDocumentRegistry([binding]); const messageId = options.messageId ?? `${runtimeId}:${binding.definition.type}:${recordKey}`; + const putMessage = () => registry.createRawDocumentPutMessage( + binding, + messageId, + recordKey, + value, + { + sourceRuntimeId: options.sourceRuntimeId ?? runtimeId, + sourceAgentId: options.sourceAgentId, + sourceRole: options.sourceRole, + tags: options.tags, + }, + ); const peer = await CultMesh.createRudpPeer(runtimeId, connectionId, endpoint, options); try { + if (options.sendOnly) { + try { + peer.send(putMessage()); + await peer.flush(options.flushTimeoutMs ?? 30_000); + } catch { + throw new CultMeshRudpWriteUnconfirmedError(messageId, "local_send_failed"); + } + return "sent"; + } // A transport ack is not admission: the put completes on its result frame, and the WriteTracker alone decides // how it settles (write-receipt.md, "Client rules"). This function only maps the settlement to resolve/throw. let receiptCapable = false; @@ -6248,18 +6278,7 @@ export class CultMesh { tracker = writeTracker; writeTracker.send(messageId, Date.now()); try { - peer.send(registry.createRawDocumentPutMessage( - binding, - messageId, - recordKey, - value, - { - sourceRuntimeId: options.sourceRuntimeId ?? runtimeId, - sourceAgentId: options.sourceAgentId, - sourceRole: options.sourceRole, - tags: options.tags, - }, - )); + peer.send(putMessage()); // The flush may not outlast the result window: a send nobody acknowledged by then is a local send failure. const windowRemaining = writeTracker.nextDeadlineMs(Date.now()) ?? 0; await peer.flush(Math.min(options.flushTimeoutMs ?? 30_000, Math.max(windowRemaining, 1))); @@ -6277,6 +6296,7 @@ export class CultMesh { } if (settlement.outcome === "refused") throw new CultMeshRudpWriteRefusedError(settlement.response); if (settlement.outcome === "unconfirmed") throw new CultMeshRudpWriteUnconfirmedError(messageId, settlement.reason); + return "admitted"; } finally { peer.close(); } diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index 19f27a3c..f1f05410 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -175,6 +175,37 @@ test("a server that does not advertise receipts and echoes the put's id back has } finally { server.close(); } }); +test("a silent server never yields admitted, in either mode: the default is unconfirmed, sendOnly is the distinct outcome sent", async () => { + const server = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); + try { + await assert.rejects(publish(server, { resultWindowMs: 200 }), CultMeshRudpWriteUnconfirmedError); + const outcome = await publish(server, { sendOnly: true }); + assert.equal(outcome, "sent"); + assert.notEqual(outcome, "admitted"); + assert.deepEqual(server.seen, ["cultnet.document_put_raw.v0", "cultnet.document_put_raw.v0"], "sendOnly sends no hello"); + } finally { server.close(); } +}); + +test("only a write_admitted resolves admitted, and sendOnly does not wait for or act on an answer", async () => { + const admitting = await fakeServer(nextConnection, { advertise: true, answer: "admit" }); + try { + assert.equal(await publish(admitting, { writeReceipts: true }), "admitted"); + } finally { admitting.close(); } + const refusing = await fakeServer(nextConnection, { advertise: true, answer: "refuse", delayMs: 1 }); + try { + assert.equal(await publish(refusing, { sendOnly: true }), "sent", "a refusal is not awaited: sendOnly says only that the put was sent"); + } finally { refusing.close(); } +}); + +test("sendOnly fails local_send_failed when the send is never acknowledged, and cannot be combined with writeReceipts", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "admit", ack: false }); + try { + await assert.rejects(publish(server, { sendOnly: true, flushTimeoutMs: 300 }), (error: unknown) => + error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "local_send_failed"); + await assert.rejects(publish(server, { sendOnly: true, writeReceipts: true }), TypeError); + } finally { server.close(); } +}); + test("without writeReceipts no hello is sent and the receipt frames are never asked for", async () => { const server = await fakeServer(nextConnection, { advertise: true, answer: "silence" }); try { diff --git a/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts b/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts index 209a2881..a5994f56 100644 --- a/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts +++ b/packages/cultnet-ts/src/idunn-odin-presence-publisher.ts @@ -52,7 +52,7 @@ export function createIdunnRuntimePresencePublisher( const publish = async (detail: string) => { const signed = signer.sign(detail); messageSequence += 1; - await publishDocument(endpoint, connectionId, { + await sendPresenceDocument(endpoint, connectionId, { schemaVersion: "cultnet.document_put_raw.v0", messageId: `runtime-presence:${signer.authority.expected.target}:${signer.runtimeInstanceId}:${messageSequence}`, document: signed.document, @@ -86,11 +86,19 @@ export function createIdunnRuntimePresencePublisher( }; } -async function publishDocument( +/** + * Sends one presence put and says only that it was sent (write-receipt.md: the explicit send-only mode). Odin does not + * answer a write with a receipt yet, so no frame, whatever id it carries, is admission; this never claims it. It returns + * once the transport has acknowledged the put, rejects `IdunnPresenceRefusedError` if a refusal arrives by then, and + * rejects if the put is never acknowledged. + * + * @internal exported for tests + */ +export async function sendPresenceDocument( endpoint: { host: string; port: number }, connectionId: number, message: Record, -): Promise { +): Promise<"sent"> { const socket = dgram.createSocket(endpoint.host.includes(":") ? "udp6" : "udp4"); await new Promise((resolve, reject) => { socket.once("error", reject); @@ -107,26 +115,24 @@ async function publishDocument( const wirePayload = encode(encodeCultNetMessageForWire(message as never, "cultnet.schema.v0")); const packets = session.sendMany("schema", wirePayload, { reliable: true, ordered: true, nowMs: Date.now() }); for (const packet of packets) await sendPacket(socket, endpoint, packet); - // The transport ack is not admission: the put completes on Odin's result frame. A refusal rejects whether - // it arrives before the ack, after it, or inside the result window; a success reply ends the wait early. - let succeeded = false; + // The transport ack is delivery, not admission. A refusal that arrives by then rejects; no other frame means anything. const onFrame = (frame: { channelId: string; payload: Uint8Array }) => { if (frame.channelId !== "schema") return; const response = parseCultNetMessage(decode(frame.payload)); if (response.schemaVersion === "cultnet.error.v0") throw new IdunnPresenceRefusedError(response); - succeeded = true; + // An error.v1 refuses this put only when it names it (or names nothing). + if (response.schemaVersion === "cultnet.error.v1" && (response.messageId === null || response.messageId === message.messageId)) { + throw new IdunnPresenceRefusedError(response); + } }; await receiveUntil(receiver, session, endpoint, () => session.outstandingReliablePacketCount === 0, 2000, onFrame); - if (!succeeded) await receiveUntil(receiver, session, endpoint, () => succeeded, RESULT_WINDOW_MS, onFrame, true); + return "sent"; } finally { receiver.close(); socket.close(); } } -/** How long after the transport ack Odin's refusal may still arrive; a success reply ends the wait early. */ -const RESULT_WINDOW_MS = 500; - /** Odin refused the presence put: the typed `cultnet.error.v0` it answered with. */ export class IdunnPresenceRefusedError extends Error { readonly code: string | undefined; @@ -200,7 +206,6 @@ async function receiveUntil( predicate: (packet: CultNetRudpPacket) => boolean, timeoutMs: number, onDelivered?: (frame: { channelId: string; payload: Uint8Array; sequence: number }) => void, - returnOnTimeout = false, ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -215,7 +220,6 @@ async function receiveUntil( } for (const packet of session.dueResends(Date.now())) await sendPacket(receiver.socket, endpoint, packet); } - if (returnOnTimeout) return; throw new Error(`Timed out waiting for CultNet RUDP response after ${timeoutMs} ms.`); } diff --git a/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts b/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts new file mode 100644 index 00000000..10a249f1 --- /dev/null +++ b/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts @@ -0,0 +1,82 @@ +// The presence publisher is send-only (write-receipt.md): Odin answers no write with a receipt yet, so no frame, whatever +// id it carries, is admission. Each test serves it a fake Odin that acknowledges the put and answers as scripted. +import assert from "node:assert/strict"; +import { createSocket, type Socket } from "node:dgram"; +import test from "node:test"; +import { encode, decode } from "@msgpack/msgpack"; + +import { encodeCultNetMessageForWire, type CultNetMessage } from "../src/contracts"; +import { IdunnPresenceRefusedError, sendPresenceDocument } from "../src/idunn-odin-presence-publisher"; +import { CultNetRudpSession, decodeRudpPacket, encodeRudpPacket } from "../src/rudp"; + +interface Script { + answer: "silence" | "echo" | "admitted" | "refuse-v0" | "refuse-v1-other"; + /** Acknowledge the put? Default true. */ + ack?: boolean; +} + +async function fakeOdin(connectionId: number, script: Script) { + const socket: Socket = createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + const session = new CultNetRudpSession({ connectionId, resendDelayMs: 50 }); + const send = (packet: Parameters[0], port: number, address: string) => { + try { socket.send(encodeRudpPacket(packet), port, address); } catch { /* closed */ } + }; + const say = (message: CultNetMessage, port: number, address: string) => { + const packets = session.sendMany("schema", encode(encodeCultNetMessageForWire(message, "cultnet.schema.v0")), { reliable: true, ordered: true, nowMs: Date.now() }); + packets.forEach((p) => send(p, port, address)); + }; + socket.on("message", (wire, remote) => { + const packet = decodeRudpPacket(wire); + const now = Date.now(); + if (packet.packetType === "connect") { + send(session.acceptConnect(packet, now), remote.port, remote.address); + return; + } + const result = session.receive(packet, now); + if (result.reply) send(result.reply, remote.port, remote.address); + for (const frame of result.delivered) { + if (frame.channelId !== "schema") continue; + const put = decode(frame.payload) as { messageId: string; document: unknown }; + if (script.answer === "echo") say({ schemaVersion: "cultnet.document_put_raw.v0", messageId: put.messageId, document: put.document } as unknown as CultNetMessage, remote.port, remote.address); + else if (script.answer === "admitted") say({ schemaVersion: "cultnet.write_admitted.v0", messageId: put.messageId, operation: "put", schemaId: "s", recordKey: "k" }, remote.port, remote.address); + else if (script.answer === "refuse-v0") say({ schemaVersion: "cultnet.error.v0", error: "no", routingHint: null, code: "write_not_authorized", details: null }, remote.port, remote.address); + else if (script.answer === "refuse-v1-other") say({ schemaVersion: "cultnet.error.v1", error: "no", routingHint: null, code: "write_not_authorized", details: null, messageId: "someone-else" }, remote.port, remote.address); + } + if (packet.reliable && script.ack !== false) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); + }); + return { port: socket.address().port, close: () => socket.close() }; +} + +let nextConnection = 0x0d1d1000; +const put = (id: string) => ({ schemaVersion: "cultnet.document_put_raw.v0", messageId: id, document: { schemaId: "s", recordKey: "k", storedAt: "t", payloadEncoding: "messagepack", payload: new Uint8Array([1]) } }); +const sendTo = (server: { port: number }, id: string, connectionId: number) => + sendPresenceDocument({ host: "127.0.0.1", port: server.port }, connectionId, put(id)); + +for (const answer of ["silence", "echo", "admitted", "refuse-v1-other"] as const) { + test(`a ${answer} Odin yields sent, never admitted: no frame carrying the put's id is success`, async () => { + const connectionId = nextConnection++; + const server = await fakeOdin(connectionId, { answer }); + try { + const outcome = await sendTo(server, "presence-1", connectionId); + assert.equal(outcome, "sent"); + assert.notEqual(outcome as string, "admitted"); + } finally { server.close(); } + }); +} + +test("a refusal that arrives by the transport ack rejects the put", async () => { + const connectionId = nextConnection++; + const server = await fakeOdin(connectionId, { answer: "refuse-v0" }); + try { + await assert.rejects(sendTo(server, "presence-1", connectionId), IdunnPresenceRefusedError); + } finally { server.close(); } +}); + +test("a put the transport never acknowledges is an error, not sent", async () => { + const connectionId = nextConnection++; + const server = await fakeOdin(connectionId, { answer: "silence", ack: false }); + try { + await assert.rejects(sendTo(server, "presence-1", connectionId), /Timed out/); + } finally { server.close(); } +}); From 206dbe142bf4c448c186d8f39c6b4d9d5b30c011 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:24:52 +0000 Subject: [PATCH 061/117] cultmesh-ts: sendOnly tests use one fake server per publish (the reused one threw on the second connect and hung the run); spec names send-only and the presence publisher The hang was the fake server, not the writer: it is bound to one connection id, and two publishes in one test sent a second connect it rejected from inside the socket handler. The unacknowledged-send test also scripted an admit, whose reply piggybacks the put's ack. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- .../test/write-receipt-client.test.ts | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index ce708ca6..f927fc00 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -61,7 +61,7 @@ A write settles as exactly one of: ## Changes to what #55 to #58 shipped -Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The TypeScript one-shot writer (#62) already reports it so; the C# and Rust clients of #55 to #58 keep the window-success until their conformance work lands, and callers of those must not read it as admission. +Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The TypeScript one-shot writer (#62) already reports it so: it resolves `admitted` only on a `write_admitted` carrying the put's id, and no other frame, a put echoed under the same id included, ever counts as admission. A caller whose server cannot answer a write yet (Odin, until it is receipt-capable) opts into the explicit send-only mode per call (`sendOnly: true`), which resolves the distinct outcome `sent` once the transport has acknowledged the put and never `admitted`; the Idunn presence publisher uses it. Receipts and send-only are both opt-in, neither is the default. The C# and Rust clients of #55 to #58 keep the window-success until their conformance work lands, and callers of those must not read it as admission. ## Vectors diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index f1f05410..797df1cd 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -176,14 +176,17 @@ test("a server that does not advertise receipts and echoes the put's id back has }); test("a silent server never yields admitted, in either mode: the default is unconfirmed, sendOnly is the distinct outcome sent", async () => { - const server = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); + const dflt = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); try { - await assert.rejects(publish(server, { resultWindowMs: 200 }), CultMeshRudpWriteUnconfirmedError); - const outcome = await publish(server, { sendOnly: true }); + await assert.rejects(publish(dflt, { resultWindowMs: 200 }), CultMeshRudpWriteUnconfirmedError); + } finally { dflt.close(); } + const sendOnly = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); + try { + const outcome = await publish(sendOnly, { sendOnly: true }); assert.equal(outcome, "sent"); assert.notEqual(outcome, "admitted"); - assert.deepEqual(server.seen, ["cultnet.document_put_raw.v0", "cultnet.document_put_raw.v0"], "sendOnly sends no hello"); - } finally { server.close(); } + assert.deepEqual(sendOnly.seen, ["cultnet.document_put_raw.v0"], "sendOnly sends no hello"); + } finally { sendOnly.close(); } }); test("only a write_admitted resolves admitted, and sendOnly does not wait for or act on an answer", async () => { @@ -197,15 +200,18 @@ test("only a write_admitted resolves admitted, and sendOnly does not wait for or } finally { refusing.close(); } }); -test("sendOnly fails local_send_failed when the send is never acknowledged, and cannot be combined with writeReceipts", async () => { - const server = await fakeServer(nextConnection, { advertise: true, answer: "admit", ack: false }); +test("sendOnly fails local_send_failed when the send is never acknowledged", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "silence", ack: false }); try { await assert.rejects(publish(server, { sendOnly: true, flushTimeoutMs: 300 }), (error: unknown) => error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "local_send_failed"); - await assert.rejects(publish(server, { sendOnly: true, writeReceipts: true }), TypeError); } finally { server.close(); } }); +test("sendOnly cannot be combined with writeReceipts", async () => { + await assert.rejects(publish({ endpoint: "rudp://127.0.0.1:9" }, { sendOnly: true, writeReceipts: true }), TypeError); +}); + test("without writeReceipts no hello is sent and the receipt frames are never asked for", async () => { const server = await fakeServer(nextConnection, { advertise: true, answer: "silence" }); try { From 89ff3d2cbd0e136f8b75faa3f4a408c978d4d934 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:26:25 +0000 Subject: [PATCH 062/117] cultnet-ts test: the Odin presence publisher is send-only, so a refusal after the transport ack is not awaited Co-Authored-By: Claude Sonnet 5.5 --- .../test/idunn-runtime-authority.test.ts | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index 2e05bbf4..50236782 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -345,18 +345,27 @@ test("the Odin publisher publishes the signer's document to the endpoint Expecte assert.equal(peer.failure, undefined, peer.failure?.message); }); -for (const order of ["ack-then-refusal", "refusal-then-ack"] as const) { - test(`the Odin publisher does not complete on the ack of a refused presence (${order})`, async (context) => { - const endpoint = expectedFacts("service").odinEndpoint!; - const peer = await startOdinPeer(context, endpoint); - peer.reject = true; - peer.order = order; - const signer = signerFor(context, "service"); - signer.reportHealth("active"); - const publisher = createIdunnRuntimePresencePublisher({ signer, endpoint }); - await assert.rejects(publisher.publish("denied presence"), /Odin rejected runtime presence: test admission denied/); - }); -} +test("the Odin publisher rejects a presence whose refusal arrives before the transport ack", async (context) => { + const endpoint = expectedFacts("service").odinEndpoint!; + const peer = await startOdinPeer(context, endpoint); + peer.reject = true; + peer.order = "refusal-then-ack"; + const signer = signerFor(context, "service"); + signer.reportHealth("active"); + const publisher = createIdunnRuntimePresencePublisher({ signer, endpoint }); + await assert.rejects(publisher.publish("denied presence"), /Odin rejected runtime presence: test admission denied/); +}); + +test("the Odin publisher is send-only: a refusal that follows the transport ack is not awaited", async (context) => { + const endpoint = expectedFacts("service").odinEndpoint!; + const peer = await startOdinPeer(context, endpoint); + peer.reject = true; + peer.order = "ack-then-refusal"; + const signer = signerFor(context, "service"); + signer.reportHealth("active"); + const publisher = createIdunnRuntimePresencePublisher({ signer, endpoint }); + await publisher.publish("denied presence"); +}); test("the Odin publisher leaves a capability shortfall to the Rust correlation instead of refusing to publish", async (context) => { const endpoint = expectedFacts("service").odinEndpoint!; From 0fbf2baea6e146a544059abb0d6bf9ed40b2e3de Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:32:49 +0000 Subject: [PATCH 063/117] cultnet-ts, cultmesh-ts: Heimdall's five Lows on send-only (spec wording, error.v1 ack-time refusal tests, build errors keep their cause, presence changelog, ack-cut anchors) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- docs/cultnet-rudp-ack-cut.md | 4 ++-- packages/cultmesh-ts/src/index.ts | 4 +++- .../cultmesh-ts/test/write-receipt-client.test.ts | 11 +++++++++++ packages/cultnet-ts/CHANGELOG.md | 1 + .../test/idunn-odin-presence-send-only.test.ts | 14 +++++++++++++- 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index f927fc00..1971b3b9 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -61,7 +61,7 @@ A write settles as exactly one of: ## Changes to what #55 to #58 shipped -Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The TypeScript one-shot writer (#62) already reports it so: it resolves `admitted` only on a `write_admitted` carrying the put's id, and no other frame, a put echoed under the same id included, ever counts as admission. A caller whose server cannot answer a write yet (Odin, until it is receipt-capable) opts into the explicit send-only mode per call (`sendOnly: true`), which resolves the distinct outcome `sent` once the transport has acknowledged the put and never `admitted`; the Idunn presence publisher uses it. Receipts and send-only are both opt-in, neither is the default. The C# and Rust clients of #55 to #58 keep the window-success until their conformance work lands, and callers of those must not read it as admission. +Those clients complete *successfully* when the window ends with no refusal. Under this spec that case is `unconfirmed`. The TypeScript one-shot writer (#62) already reports it so: it resolves `admitted` only on a `write_admitted` carrying the put's id, and no other frame, a put echoed under the same id included, ever counts as admission. A caller whose server cannot answer a write yet (Odin, until it is receipt-capable) opts into the explicit send-only mode per call (`sendOnly: true`), which resolves the distinct outcome `sent` once the transport has acknowledged the put and never `admitted`. The Idunn presence publisher is send-only by its own function (`sendPresenceDocument`, not the `sendOnly` option): the same `sent` result, plus a rejection if a refusal naming the put (or naming nothing) arrives by the transport ack. Receipts and send-only are both opt-in, neither is the default. The C# and Rust clients of #55 to #58 keep the window-success until their conformance work lands, and callers of those must not read it as admission. ## Vectors diff --git a/docs/cultnet-rudp-ack-cut.md b/docs/cultnet-rudp-ack-cut.md index 1b519484..47008c81 100644 --- a/docs/cultnet-rudp-ack-cut.md +++ b/docs/cultnet-rudp-ack-cut.md @@ -441,8 +441,8 @@ consumer that waits wants delivered, which is now just Acknowledged. | `cultmesh-rs/src/lib.rs:781-838` `request_raw_snapshot_from_rudp_catalog` (Idunn `src/drivers.rs:5527` uses it) | a hand-rolled loop until `Acknowledged`, then the response | Cut 4: `wait_acknowledged` | | `cultmesh-rs/src/lib.rs:870-900` `publish_cultnet_message_to_rudp_catalog` (Odin `odin-daemon`) | a hand-rolled loop until `Acknowledged`, then disconnect. The doc at `:869` claims "application admission" | Cut 4: `wait_acknowledged`; the doc says delivered to the document server's sink (`rudp_document_server.rs:343-372`) | | `cultmesh-rs/src/lib.rs:918-938` `publish_cultnet_messages_to_rudp_catalog` (Ghostlight `mesh.rs:323`) | `flush_reliable`, then drop. **Several ordered frames, then close** | None; flush now means it | -| `cultmesh-ts/src/index.ts:6070-6098` `publishRudpDocumentOnce` (Heimdall, StreamPixels, Stonks, weksa, Bifrost, VoidBot, Hermodr) | `peer.flush` (`cultnet-ts/src/peer.ts:109-110`), then close | None. Cut 5 moves its connect-and-wait helper (`index.ts:6603-6650`) into cultnet-ts | -| `cultnet-ts/src/idunn-odin-presence-publisher.ts:89-215` `publishDocument`, behind `createIdunnRuntimePresencePublisher`. **StreamPixels' service runs this**: `apps/service/scripts/idunn-entry.mjs` runs `deployment/idunn/runtime-presence.mjs`, which calls `createIdunnRuntimePresencePublisher` (`:46`) from its vendored CultLib `542ddd3` (StreamPixels `a49f7d8`; pass 2's `30ee8b9` is stale), where it lives under its older name `cultnet-ts/src/idunn-runtime-authority.ts:206` | a hand-rolled session loop (`:186-208`) until `outstandingReliablePacketCount === 0`. It does resend (`:205`). It drops `readyToSend` (`:198`), so promoted queued packets wait for the resend timer. It pins `initialSequence: 1` (`:103`) and never sends a Disconnect, so Odin's hub keeps each publish's peer until it times out (each publish binds a new port, so each is a new peer) | Cut 5: the socket transport, `connectAndWait`, `send("schema")`, `flush`, `close` (which sends the Disconnect, `rudp.ts:956-969`) | +| `cultmesh-ts/src/index.ts:6211-6303` `publishRudpDocumentOnce` (Heimdall, StreamPixels, Stonks, weksa, Bifrost, VoidBot, Hermodr) | `peer.flush` (`cultnet-ts/src/peer.ts:109-110`), then close | None. Cut 5 moves its connect-and-wait helper (`index.ts:6603-6650`) into cultnet-ts | +| `cultnet-ts/src/idunn-odin-presence-publisher.ts:97-136` `sendPresenceDocument` (formerly `publishDocument`), behind `createIdunnRuntimePresencePublisher`. **StreamPixels' service runs this**: `apps/service/scripts/idunn-entry.mjs` runs `deployment/idunn/runtime-presence.mjs`, which calls `createIdunnRuntimePresencePublisher` (`:46`) from its vendored CultLib `542ddd3` (StreamPixels `a49f7d8`; pass 2's `30ee8b9` is stale), where it lives under its older name `cultnet-ts/src/idunn-runtime-authority.ts:206` | a hand-rolled session loop (`:186-208`) until `outstandingReliablePacketCount === 0`. It does resend (`:205`). It drops `readyToSend` (`:198`), so promoted queued packets wait for the resend timer. It pins `initialSequence: 1` (`:103`) and never sends a Disconnect, so Odin's hub keeps each publish's peer until it times out (each publish binds a new port, so each is a new peer) | Cut 5: the socket transport, `connectAndWait`, `send("schema")`, `flush`, `close` (which sends the Disconnect, `rudp.ts:956-969`) | | `cultnet-ts/src/signed-daemon-health.ts:104-162` `publishSignedDaemonHealth` (Hermodr, Odin `idunn-rudp.cjs`) | a hand-rolled loop: sends the packets `sendMany` admitted, waits up to 500 ms for the **first** Ack packet of any content (`:146-153`), swallows the timeout, closes. Resends run inside that wait (`:303`) but promoted queued packets are never sent | Cut 5, as above. Today a write not acked within 500 ms, or any payload over 32 packets, is reported published | | `cultnet-ts/src/operation-service.ts` | the operation response | none (the response is the proof) | | C# `FlushReliable` callers: Gjallar `Program.cs:3055-3080` (two ordered frames, flush, dispose) | flush | None | diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 38c0b34a..0bf8ceae 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -6237,8 +6237,10 @@ export class CultMesh { const peer = await CultMesh.createRudpPeer(runtimeId, connectionId, endpoint, options); try { if (options.sendOnly) { + // Built outside the try: a message that cannot be serialized is the caller's error, with its cause, not a send failure. + const put = putMessage(); try { - peer.send(putMessage()); + peer.send(put); await peer.flush(options.flushTimeoutMs ?? 30_000); } catch { throw new CultMeshRudpWriteUnconfirmedError(messageId, "local_send_failed"); diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index 797df1cd..b5f67de6 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -208,6 +208,17 @@ test("sendOnly fails local_send_failed when the send is never acknowledged", asy } finally { server.close(); } }); +test("sendOnly keeps the cause of a put that cannot be built: it is not reported as local_send_failed", async () => { + const server = await fakeServer(nextConnection, { advertise: false, answer: "silence" }); + try { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("receipt-client", nextConnection++, server.endpoint, binding, "tests:receipt:a", { noteId: 1 } as unknown as { noteId: string; text: string }, { ...transport, sendOnly: true }), + (error: unknown) => !(error instanceof CultMeshRudpWriteUnconfirmedError) && error instanceof Error, + ); + assert.deepEqual(server.seen, [], "nothing was sent"); + } finally { server.close(); } +}); + test("sendOnly cannot be combined with writeReceipts", async () => { await assert.rejects(publish({ endpoint: "rudp://127.0.0.1:9" }, { sendOnly: true, writeReceipts: true }), TypeError); }); diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index 13dc3dff..be6e2d98 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to `@gamecult/cultnet-ts` are recorded here, following ### Breaking +- The Idunn runtime presence publisher is send-only (`sendPresenceDocument`): it returns `"sent"` once Odin's transport has acknowledged the put and no longer waits up to 500 ms for a success reply, which was never admission. It rejects `IdunnPresenceRefusedError` for a `cultnet.error.v0`, or a `cultnet.error.v1` naming the put or nothing, that arrives by the ack; a refusal after the ack is not awaited, and no other frame, one carrying the put's id included, counts as success. - The package is renamed `cultnet-ts` to `@gamecult/cultnet-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultnet-ts` and its subpaths; the API is unchanged. - `CultNetErrorMessage` carries `routingHint`, `code` and `details` (null when absent), as the reference writes them and `cultnet.error.schema.json` requires; the interface previously dropped them. New `CultNetErrorCode`, `CultNetErrorDetails` and `CultNetShardRoutingHint` types. diff --git a/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts b/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts index 10a249f1..fff8e2d5 100644 --- a/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts +++ b/packages/cultnet-ts/test/idunn-odin-presence-send-only.test.ts @@ -10,7 +10,7 @@ import { IdunnPresenceRefusedError, sendPresenceDocument } from "../src/idunn-od import { CultNetRudpSession, decodeRudpPacket, encodeRudpPacket } from "../src/rudp"; interface Script { - answer: "silence" | "echo" | "admitted" | "refuse-v0" | "refuse-v1-other"; + answer: "silence" | "echo" | "admitted" | "refuse-v0" | "refuse-v1-other" | "refuse-v1-same" | "refuse-v1-none"; /** Acknowledge the put? Default true. */ ack?: boolean; } @@ -42,6 +42,8 @@ async function fakeOdin(connectionId: number, script: Script) { else if (script.answer === "admitted") say({ schemaVersion: "cultnet.write_admitted.v0", messageId: put.messageId, operation: "put", schemaId: "s", recordKey: "k" }, remote.port, remote.address); else if (script.answer === "refuse-v0") say({ schemaVersion: "cultnet.error.v0", error: "no", routingHint: null, code: "write_not_authorized", details: null }, remote.port, remote.address); else if (script.answer === "refuse-v1-other") say({ schemaVersion: "cultnet.error.v1", error: "no", routingHint: null, code: "write_not_authorized", details: null, messageId: "someone-else" }, remote.port, remote.address); + else if (script.answer === "refuse-v1-same") say({ schemaVersion: "cultnet.error.v1", error: "no", routingHint: null, code: "write_not_authorized", details: null, messageId: put.messageId }, remote.port, remote.address); + else if (script.answer === "refuse-v1-none") say({ schemaVersion: "cultnet.error.v1", error: "no", routingHint: null, code: "write_not_authorized", details: null, messageId: null }, remote.port, remote.address); } if (packet.reliable && script.ack !== false) send(session.createAckForReceived(packet.sequence), remote.port, remote.address); }); @@ -73,6 +75,16 @@ test("a refusal that arrives by the transport ack rejects the put", async () => } finally { server.close(); } }); +for (const answer of ["refuse-v1-same", "refuse-v1-none"] as const) { + test(`an error.v1 refusal that ${answer === "refuse-v1-same" ? "names this put" : "names no put"} rejects the put`, async () => { + const connectionId = nextConnection++; + const server = await fakeOdin(connectionId, { answer }); + try { + await assert.rejects(sendTo(server, "presence-1", connectionId), IdunnPresenceRefusedError); + } finally { server.close(); } + }); +} + test("a put the transport never acknowledges is an error, not sent", async () => { const connectionId = nextConnection++; const server = await fakeOdin(connectionId, { answer: "silence", ack: false }); From dc1ae51e928b6b1d9c03a13a379849f8e54d5786 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:37:08 +0000 Subject: [PATCH 064/117] cultmesh-ts: publishRudpDocumentOnce builds the put before connecting, so a put that cannot be built keeps its cause on both paths Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/src/index.ts | 7 +++---- .../cultmesh-ts/test/write-receipt-client.test.ts | 11 +++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 0bf8ceae..2f734cfd 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -6222,7 +6222,8 @@ export class CultMesh { if (options.sendOnly && options.writeReceipts) throw new TypeError("sendOnly and writeReceipts cannot be combined."); const registry = new CultNetDocumentRegistry([binding]); const messageId = options.messageId ?? `${runtimeId}:${binding.definition.type}:${recordKey}`; - const putMessage = () => registry.createRawDocumentPutMessage( + // Built before any connection and outside every try: a put that cannot be built is the caller's error, with its cause. + const put = registry.createRawDocumentPutMessage( binding, messageId, recordKey, @@ -6237,8 +6238,6 @@ export class CultMesh { const peer = await CultMesh.createRudpPeer(runtimeId, connectionId, endpoint, options); try { if (options.sendOnly) { - // Built outside the try: a message that cannot be serialized is the caller's error, with its cause, not a send failure. - const put = putMessage(); try { peer.send(put); await peer.flush(options.flushTimeoutMs ?? 30_000); @@ -6280,7 +6279,7 @@ export class CultMesh { tracker = writeTracker; writeTracker.send(messageId, Date.now()); try { - peer.send(putMessage()); + peer.send(put); // The flush may not outlast the result window: a send nobody acknowledged by then is a local send failure. const windowRemaining = writeTracker.nextDeadlineMs(Date.now()) ?? 0; await peer.flush(Math.min(options.flushTimeoutMs ?? 30_000, Math.max(windowRemaining, 1))); diff --git a/packages/cultmesh-ts/test/write-receipt-client.test.ts b/packages/cultmesh-ts/test/write-receipt-client.test.ts index b5f67de6..072b89a3 100644 --- a/packages/cultmesh-ts/test/write-receipt-client.test.ts +++ b/packages/cultmesh-ts/test/write-receipt-client.test.ts @@ -219,6 +219,17 @@ test("sendOnly keeps the cause of a put that cannot be built: it is not reported } finally { server.close(); } }); +test("the receipt path keeps the cause of a put that cannot be built: it is not reported as local_send_failed", async () => { + const server = await fakeServer(nextConnection, { advertise: true, answer: "admit" }); + try { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("receipt-client", nextConnection++, server.endpoint, binding, "tests:receipt:a", { noteId: 1 } as unknown as { noteId: string; text: string }, { ...transport, writeReceipts: true }), + (error: unknown) => !(error instanceof CultMeshRudpWriteUnconfirmedError) && error instanceof Error, + ); + assert.ok(!server.seen.includes("cultnet.document_put_raw.v0"), "no put was sent"); + } finally { server.close(); } +}); + test("sendOnly cannot be combined with writeReceipts", async () => { await assert.rejects(publish({ endpoint: "rudp://127.0.0.1:9" }, { sendOnly: true, writeReceipts: true }), TypeError); }); From 90d252735f422cbc0ae3c29000a005a5adb10876 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:38:43 +0000 Subject: [PATCH 065/117] cultmesh-rs: a refused reliable-sequenced write is received and acknowledged first, and a refused write counts as session activity Refusing before session.receive left the packet unacknowledged: the client resent it forever and every later ordered write stalled. The test now uses the same peer and proves a later ordered write is admitted. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 26 ++++++++++---- .../cultmesh-rs/tests/rudp_document_server.rs | 36 ++++++++++++++++--- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 4df5cd31..f97b2578 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -753,15 +753,19 @@ where self.packets_dropped += 1; return Ok(CultMeshRudpPollOutcome::Handled); } - if packet.packet_type == CultNetRudpPacketType::Data - && packet.channel_id == "schema" - && (!packet.reliable || (packet.sequenced && self.receipts(key))) - { - // A receipt session's write that could be lost without trace (unreliable, or on a sequenced channel - // that drops superseded packets) cannot be answered once: it is refused, never applied. Anything - // else unreliable is dropped as before. + let is_schema_data = packet.packet_type == CultNetRudpPacketType::Data && packet.channel_id == "schema"; + if is_schema_data && !packet.reliable { + // An unreliable write cannot be answered once: a receipt session is refused, never applied, and + // anything else unreliable is dropped as before. Either way the peer is not idle. + if let Some(entry) = self.sessions.get_mut(&key) { + entry.last_activity_monotonic_millis = now_monotonic; + } return self.refuse_unreliable_write(key, &packet.payload); } + // A receipt session's write on a sequenced channel (which drops superseded packets) is refused too, but + // only after the session has received and acknowledged it: an unacknowledged packet is resent forever + // and stalls every later ordered write. + let refuse_sequenced = is_schema_data && packet.sequenced && self.receipts(key); let payload_charge = if packet.packet_type == CultNetRudpPacketType::Data { packet.payload.len() } else { @@ -821,6 +825,14 @@ where if frame.channel_id != "schema" { continue; } + if refuse_sequenced { + if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = + self.refuse_unreliable_write(key, &frame.payload)? + { + kept_rejection = Some(rejection); + } + continue; + } // The capped frame was answered: nothing after it is served while the answer is delivered. if self .sessions diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index ac43c21d..cccff92f 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2963,11 +2963,8 @@ fn only_reliable_non_sequenced_writes_are_accepted_on_a_receipt_session() -> Res let unordered = send_with(&mut peer, &mut server, "s1", false, false)?; assert!(unordered.contains(&admitted("s1", "note:1")), "reliable unordered is accepted, and the id was not consumed: {unordered:?}"); - // A fresh peer: an ordered stream waits on the sequence numbers the refused packet used. - let mut ordered_peer = RawPeer::connect(&mut server, 251)?; - ordered_peer.send(&receipt_hello())?; - server.poll_once()?; - let ordered = send_with(&mut ordered_peer, &mut server, "o1", true, false)?; + // The same peer: the refused packet was acknowledged, so an ordered stream does not wait on its sequence number. + let ordered = send_with(&mut peer, &mut server, "o1", true, false)?; assert!(ordered.contains(&admitted("o1", "note:1")), "reliable ordered is accepted: {ordered:?}"); assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); @@ -2986,6 +2983,35 @@ fn only_reliable_non_sequenced_writes_are_accepted_on_a_receipt_session() -> Res Ok(()) } +/// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured +/// from the refusal and not from the packet before it. +#[test] +fn a_refused_write_keeps_the_session_from_going_idle() -> Result<()> { + let clock = Clock::new(70_000); + let options = CultMeshRudpDocumentServerOptions { session_idle_timeout: Duration::from_secs(1), ..Default::default() }; + let mut server = server(options, clock.clone(), Sink::default(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 252)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + for (at, reliable, sequenced) in [(70_700, false, false), (71_400, true, true)] { + clock.set(at); + let payload = encode_cultnet_message_to_vec(&put("idle", "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable, ordered: false, sequenced, now_ms: at, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + assert_eq!(server.maintain()?.sessions_expired, 0, "the refused write at {at} is activity"); + } + assert_eq!(server.session_count(), 1); + Ok(()) +} + /// A Connect that does not repeat the accepted one is a new client session: it starts with nothing remembered, so /// the id a receipt session already used is a new write on it, and the old session's entries leave the budget. #[test] From 621b86ca15f8d7f79a03e6cdd1c4d2808a1d9abd Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:45:19 +0000 Subject: [PATCH 066/117] Spec #59: an empty or blank schemaId or recordKey is message_invalid (details.field message, the write's id), never applied; vector for it Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet/interop/write-receipt-vectors.json | 17 +++++++++++++++++ contracts/cultnet/interop/write-receipt.md | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index f575398d..c0313569 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -75,6 +75,23 @@ "messageId": null } }, + { + "name": "error_v1_empty_schema_id_is_unreadable_and_names_the_write", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D9216D6573736167655F696E76616C69643A20756E7265616461626C65206672616D65AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D31", + "message": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid: unreadable frame", + "routingHint": null, + "code": "message_invalid", + "details": { + "field": "message", + "value": null, + "asOf": null, + "current": null + }, + "messageId": "put-1" + } + }, { "name": "error_v1_limit_reached", "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93B6C696D69745F726561636865643A207468652073657373696F6E27732072656D656D62657265642D7772697465206C696D69742069732066756C6CAB726F7574696E6748696E74C0A4636F6465AD6C696D69745F72656163686564A764657461696C7384A56669656C64C0A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A87075742D31303235", diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index ce708ca6..75da775f 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -36,7 +36,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. + - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A `schemaId` or `recordKey` that is empty or only whitespace (present but names nothing) is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"` and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. From 1a228349f15d9b7ad0b628ffafc2820b91bebda4 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 19:46:45 +0000 Subject: [PATCH 067/117] C#: a put or delete with an empty or blank schemaId or recordKey is refused message_invalid (as an unreadable frame, like the Rust server) and not applied Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetDatabaseServer.cs | 20 ++++++++++ .../CultNetWriteReceiptServerTests.cs | 37 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 16967d49..1746391e 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -391,6 +391,14 @@ namespace GameCult.Networking } // A refusal: error.v1 carrying the write's id for a receipt-capable connection, the v0 error for any other. + // An empty or blank schemaId or recordKey names no record: refused as a frame that does not read, exactly as the Rust + // server's decoder refuses it (write-receipt.md, "Required fields"), never handed to the store. + private static bool IsUnreadableWriteTarget(string? schemaId, string? recordKey) => + string.IsNullOrWhiteSpace(schemaId) || string.IsNullOrWhiteSpace(recordKey); + + private static CultNetErrorMessage UnreadableWriteTarget() => + CultNetErrorMessage.ForMessageInvalid("message", "the message could not be read"); + private static void Refuse(CultNetServerPeer peer, CultNetReceiptedWrite? write, CultNetErrorMessage error) { if (write != null) write.Refused(error); @@ -431,6 +439,12 @@ namespace GameCult.Networking return; } + if (IsUnreadableWriteTarget(message.Document!.SchemaId, message.Document.RecordKey)) + { + Refuse(peer, write, UnreadableWriteTarget()); + return; + } + if (write != null && !write.Begin()) return; var authorize = Authorize(peer, CultNetRemoteWriteOperation.Put, message.ShardId, message.ShardEpoch); @@ -506,6 +520,12 @@ namespace GameCult.Networking return; } + if (IsUnreadableWriteTarget(message.SchemaId, message.RecordKey)) + { + Refuse(peer, write, UnreadableWriteTarget()); + return; + } + if (write != null && !write.Begin()) return; var authorize = Authorize(peer, CultNetRemoteWriteOperation.Delete, message.ShardId, message.ShardEpoch); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 13aab811..c572cfbf 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1010,6 +1010,43 @@ namespace GameCult.Networking.Tests Assert.That(error.Error, Is.EqualTo("message_invalid: schemaId is required")); } + // An empty or blank schemaId or recordKey is refused like a frame that does not read (the Rust server's rule): message_invalid, + // details.field "message", the write's id, nothing applied. A null schemaId stays "schemaId is required". + [TestCase("", null)] + [TestCase(" ", null)] + [TestCase(null, "")] + [TestCase(null, " ")] + public async Task APutWithAnEmptyOrBlankSchemaIdOrRecordKeyIsRefusedAsUnreadableAndNotApplied(string? schemaId, string? recordKey) + { + var (rig, wire) = await Capable(); + var message = rig.PutOf("m1", KeyA, "one"); + if (schemaId != null) message.Document.SchemaId = schemaId; + if (recordKey != null) message.Document.RecordKey = recordKey; + wire.Sent.Clear(); + + await rig.PutHandler(message, wire.Peer); + + var error = AssertRefused(wire.Sent.Single(), "m1", "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("message")); + Assert.That(rig.TextOf(KeyA), Is.Null); + } + + [TestCase("", "k")] + [TestCase(" ", "k")] + [TestCase("note.v0", "")] + public async Task ADeleteWithAnEmptyOrBlankSchemaIdOrRecordKeyIsRefusedAsUnreadable(string schemaId, string recordKey) + { + var (rig, wire) = await Capable(); + var message = rig.DeleteOf("d1", new CultRecordKey(recordKey)); + message.SchemaId = schemaId; + wire.Sent.Clear(); + + await rig.DeleteHandler(message, wire.Peer); + + var error = AssertRefused(wire.Sent.Single(), "d1", "message_invalid"); + Assert.That(error.Details!.Field, Is.EqualTo("message")); + } + [Test] public async Task ARefusalAtTheCapIsRememberedSoAResendReplaysItAndADifferentContentResendIsRefused() { From e99e9d4bc0824c7ff9f7317272e31be6fe7a008c Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:26:49 +0000 Subject: [PATCH 068/117] write-receipt spec: one unreadable text, one blank-field rule; Rust server tests pin both The unreadable-frame text is 'message_invalid: the message could not be read' (the servers' wording; the Rust wire never carried decoder text, only its local rejection display). A put that names nothing (blank schemaId/recordKey/storedAt, empty payload, blank present source ids or tags) is refused as unreadable on every connection; the vectors' blankFields section lists the cases and the Rust server tests replay them. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 29 +++- contracts/cultnet/interop/write-receipt.md | 6 +- .../cultmesh-rs/tests/rudp_document_server.rs | 134 ++++++++++++++++++ packages/cultnet-rs/tests/write_receipt.rs | 2 +- 4 files changed, 163 insertions(+), 8 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index c0313569..1dc39fd8 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -60,10 +60,10 @@ }, { "name": "error_v1_unreadable_frame_has_null_message_id", - "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D9216D6573736167655F696E76616C69643A20756E7265616461626C65206672616D65AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964C0", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D92E6D6573736167655F696E76616C69643A20746865206D65737361676520636F756C64206E6F742062652072656164AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964C0", "message": { "schemaVersion": "cultnet.error.v1", - "error": "message_invalid: unreadable frame", + "error": "message_invalid: the message could not be read", "routingHint": null, "code": "message_invalid", "details": { @@ -77,10 +77,10 @@ }, { "name": "error_v1_empty_schema_id_is_unreadable_and_names_the_write", - "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D9216D6573736167655F696E76616C69643A20756E7265616461626C65206672616D65AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D31", + "messagePackHex": "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D92E6D6573736167655F696E76616C69643A20746865206D65737361676520636F756C64206E6F742062652072656164AB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A76D657373616765A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D31", "message": { "schemaVersion": "cultnet.error.v1", - "error": "message_invalid: unreadable frame", + "error": "message_invalid: the message could not be read", "routingHint": null, "code": "message_invalid", "details": { @@ -110,6 +110,27 @@ } } ], + "blankFields": { + "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or only whitespace. Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.", + "validPutPayloadHex": "81A16101", + "cases": [ + { "name": "valid_document_is_admitted", "set": {}, "outcome": "admitted" }, + { "name": "empty_schema_id", "set": { "schemaId": "" }, "outcome": "refused" }, + { "name": "blank_schema_id", "set": { "schemaId": " " }, "outcome": "refused" }, + { "name": "empty_record_key", "set": { "recordKey": "" }, "outcome": "refused" }, + { "name": "blank_record_key", "set": { "recordKey": " " }, "outcome": "refused" }, + { "name": "empty_stored_at", "set": { "storedAt": "" }, "outcome": "refused" }, + { "name": "blank_stored_at", "set": { "storedAt": " " }, "outcome": "refused" }, + { "name": "empty_payload", "set": { "payloadHex": "" }, "outcome": "refused" }, + { "name": "blank_source_runtime_id", "set": { "sourceRuntimeId": "" }, "outcome": "refused" }, + { "name": "blank_source_agent_id", "set": { "sourceAgentId": " " }, "outcome": "refused" }, + { "name": "blank_source_role", "set": { "sourceRole": "" }, "outcome": "refused" }, + { "name": "blank_tag", "set": { "tags": ["a", " "] }, "outcome": "refused" }, + { "name": "source_ids_and_tags_present_and_named", "set": { "sourceRuntimeId": "r", "sourceAgentId": "a", "sourceRole": "w", "tags": ["a"] }, "outcome": "admitted" }, + { "name": "source_ids_and_tags_nil", "set": { "sourceRuntimeId": null, "sourceAgentId": null, "sourceRole": null, "tags": null }, "outcome": "admitted" }, + { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" } + ] + }, "schemaInvalid": [ { "name": "write_admitted_with_an_error_key", diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 8581cc53..35756ecf 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -36,8 +36,8 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A `schemaId` or `recordKey` that is empty or only whitespace (present but names nothing) is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"` and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. - - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. + - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is empty or only whitespace; on a put's document, also a `storedAt` that is empty or only whitespace, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and empty or only whitespace, and a `tags` entry that is empty or only whitespace. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. + - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. @@ -65,7 +65,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance status diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index cccff92f..b8772eaf 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3122,3 +3122,137 @@ fn an_unreliable_refusal_that_cannot_be_queued_ends_the_session_as_a_local_failu assert_eq!(goodbye.payload, b"server could not answer a write".to_vec()); Ok(()) } + +fn vectors() -> serde_json::Value { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +/// The spec's one fixed text for a frame that does not read or a write that names nothing, read from the vector +/// that carries it, so a server and the spec cannot drift apart unseen. +fn unreadable_vector_text(vectors: &serde_json::Value) -> String { + let name = "error_v1_empty_schema_id_is_unreadable_and_names_the_write"; + let vector = vectors["wire"].as_array().unwrap().iter().find(|v| v["name"] == name).unwrap(); + vector["message"]["error"].as_str().unwrap().to_string() +} + +/// The put frame of a `blankFields` case: the valid document with the case's `set` applied. Serialized without the +/// typed encoder, because that refuses what the case is about. +fn blank_field_put(vectors: &serde_json::Value, set: &serde_json::Value) -> Vec { + #[derive(serde::Serialize)] + #[serde(rename_all = "camelCase")] + struct Put { + schema_version: &'static str, + message_id: &'static str, + document: CultNetRawDocumentRecord, + } + let hex = set.get("payloadHex").and_then(|v| v.as_str()).unwrap_or(vectors["blankFields"]["validPutPayloadHex"].as_str().unwrap()); + let field = |name: &str, default: &str| set.get(name).and_then(|v| v.as_str()).unwrap_or(default).to_string(); + let optional = |name: &str| set.get(name).and_then(|v| v.as_str()).map(str::to_string); + let document = CultNetRawDocumentRecord { + schema_id: field("schemaId", "note.v0"), + record_key: field("recordKey", "note:1"), + stored_at: field("storedAt", "2026-11-08T00:00:00Z"), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: (0..hex.len()).step_by(2).map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()).collect(), + source_runtime_id: optional("sourceRuntimeId"), + source_agent_id: optional("sourceAgentId"), + source_role: optional("sourceRole"), + tags: set.get("tags").and_then(|v| v.as_array()).map(|tags| tags.iter().map(|t| t.as_str().unwrap().to_string()).collect()), + }; + rmp_serde::to_vec_named(&Put { schema_version: "cultnet.document_put_raw.v0", message_id: "m1", document }).unwrap() +} + +/// Each `blankFields` case: a receipt-capable connection is answered with the admitted receipt, or with the one fixed +/// unreadable text naming the write, and the sink sees a document only when the case is admitted. +#[test] +fn a_put_that_names_nothing_is_refused_as_unreadable_naming_the_write_and_a_named_one_is_admitted() -> Result<()> { + let vectors = vectors(); + let fixed = unreadable_vector_text(&vectors); + assert_eq!(fixed, "message_invalid: the message could not be read"); + let cases = vectors["blankFields"]["cases"].as_array().unwrap(); + assert!(cases.len() >= 10); + for (index, case) in cases.iter().enumerate() { + let name = case["name"].as_str().unwrap(); + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 400 + index as u32)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + peer.send_reliable("schema", blank_field_put(&vectors, &case["set"]))?; + let answers = answers_after(&mut server, &mut peer, 1)?; + match case["outcome"].as_str().unwrap() { + "admitted" => { + assert_eq!( + answers, + vec![CultNetMessage::WriteAdmitted { + message_id: "m1".into(), + operation: "put".into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + }], + "{name}" + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "{name}"); + } + "refused" => { + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("m1".into())), "{name}"); + assert_eq!(error_text_and_field(&answers[0]), (fixed.clone(), Some("message".into())), "{name}"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "{name}: never applied"); + } + other => panic!("{name}: unknown outcome {other}"), + } + assert_eq!(server.session_count(), 1, "{name}: the session is kept"); + } + Ok(()) +} + +/// The same cases on a connection that never opted in: the same refusal as `error.v0`, the same text, nothing applied. +#[test] +fn a_put_that_names_nothing_is_refused_the_same_way_on_a_legacy_connection() -> Result<()> { + let vectors = vectors(); + let fixed = unreadable_vector_text(&vectors); + for (index, case) in vectors["blankFields"]["cases"].as_array().unwrap().iter().enumerate() { + let name = case["name"].as_str().unwrap(); + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 500 + index as u32)?; + connect(&mut server, &mut [&mut peer])?; + let bytes = blank_field_put(&vectors, &case["set"]); + if case["outcome"] == "refused" { + let (code, details, error) = refusal_on_live_session(&mut server, &mut peer, bytes)?; + assert_eq!(code, Some(CultNetErrorCode::MessageInvalid), "{name}"); + assert_eq!(details.and_then(|d| d.field), Some("message".into()), "{name}"); + assert_eq!(error, fixed, "{name}"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "{name}"); + } else { + peer.send_reliable("schema", bytes)?; + for _ in 0..2_000 { + server.poll_once()?; + if !sink.0.lock().unwrap().receipts.is_empty() { + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "{name}"); + } + } + Ok(()) +} + +/// An unreadable frame from a receipt connection is sent the one fixed text, with no decoder wording in it. +#[test] +fn an_unreadable_frame_on_a_receipt_connection_is_sent_exactly_the_fixed_text() -> Result<()> { + let vectors = vectors(); + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 450)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + peer.send_reliable("schema", garbage())?; + let answers = answers_after(&mut server, &mut peer, 1)?; + assert_eq!( + error_text_and_field(&answers[0]), + (unreadable_vector_text(&vectors), Some("message".into())) + ); + Ok(()) +} diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 6d06a68c..3d233a61 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -39,7 +39,7 @@ fn the_new_frames_decode_back_equal() { ); // error.v1 with a nil id round-trips as nil, and the code enum is v0's. let error = CultNetMessage::ErrorV1 { - error: "message_invalid: unreadable frame".into(), + error: "message_invalid: the message could not be read".into(), code: Some(CultNetErrorCode::MessageInvalid), details: None, message_id: None, From bc9a1842f5587495aa03e6636991c7f7e3c12b15 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:26:49 +0000 Subject: [PATCH 069/117] C# database server: refuse a put that names nothing, one unreadable text IsUnreadableDocument follows the spec's Required fields rule (blank storedAt, empty payload, blank source ids and tags join blank schemaId/recordKey); ForUnreadableMessage is the single owner of the fixed text. Tests replay the blankFields vectors on receipt and legacy connections. Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetDatabaseServer.cs | 25 ++++-- .../CultNetSchemaMessages.cs | 7 ++ src/GameCult.Networking/CultNetTransport.cs | 2 +- .../CultNetWriteReceiptServerTests.cs | 79 +++++++++++++++++++ 4 files changed, 104 insertions(+), 9 deletions(-) diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 1746391e..31822ade 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -390,15 +390,24 @@ namespace GameCult.Networking } } - // A refusal: error.v1 carrying the write's id for a receipt-capable connection, the v0 error for any other. - // An empty or blank schemaId or recordKey names no record: refused as a frame that does not read, exactly as the Rust - // server's decoder refuses it (write-receipt.md, "Required fields"), never handed to the store. + // A write that names nothing is refused as a frame that does not read, exactly as the Rust server's decoder refuses it + // (write-receipt.md, "Required fields"), never handed to the store: a blank schemaId, recordKey or storedAt, an empty + // payload, and a source id or tag that is present but blank. private static bool IsUnreadableWriteTarget(string? schemaId, string? recordKey) => string.IsNullOrWhiteSpace(schemaId) || string.IsNullOrWhiteSpace(recordKey); - private static CultNetErrorMessage UnreadableWriteTarget() => - CultNetErrorMessage.ForMessageInvalid("message", "the message could not be read"); + private static bool IsUnreadableDocument(CultNetRawDocumentRecord document) => + IsUnreadableWriteTarget(document.SchemaId, document.RecordKey) + || string.IsNullOrWhiteSpace(document.StoredAt) + || document.Payload == null || document.Payload.Length == 0 + || IsPresentButBlank(document.SourceRuntimeId) + || IsPresentButBlank(document.SourceAgentId) + || IsPresentButBlank(document.SourceRole) + || document.Tags != null && document.Tags.Any(tag => string.IsNullOrWhiteSpace(tag)); + private static bool IsPresentButBlank(string? value) => value != null && string.IsNullOrWhiteSpace(value); + + // A refusal: error.v1 carrying the write's id for a receipt-capable connection, the v0 error for any other. private static void Refuse(CultNetServerPeer peer, CultNetReceiptedWrite? write, CultNetErrorMessage error) { if (write != null) write.Refused(error); @@ -439,9 +448,9 @@ namespace GameCult.Networking return; } - if (IsUnreadableWriteTarget(message.Document!.SchemaId, message.Document.RecordKey)) + if (IsUnreadableDocument(message.Document!)) { - Refuse(peer, write, UnreadableWriteTarget()); + Refuse(peer, write, CultNetErrorMessage.ForUnreadableMessage()); return; } @@ -522,7 +531,7 @@ namespace GameCult.Networking if (IsUnreadableWriteTarget(message.SchemaId, message.RecordKey)) { - Refuse(peer, write, UnreadableWriteTarget()); + Refuse(peer, write, CultNetErrorMessage.ForUnreadableMessage()); return; } diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index 090c6a45..a2211672 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -587,6 +587,13 @@ namespace GameCult.Networking Details = new CultNetErrorDetails { Field = field } }; + /// + /// The refusal of a frame that does not read, or whose write names nothing (write-receipt.md, Fixed texts): + /// message_invalid, details { field: "message" }, and the one fixed text. A decoder's own text never reaches the wire. + /// + public static CultNetErrorMessage ForUnreadableMessage() => + ForMessageInvalid("message", "the message could not be read"); + /// /// Builds the wire error for a cursor refusal (R-N): code cursor_stale with details /// { asOf, current }, or code cursor_invalid with no details. diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index fc41b2f4..d84b0263 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3571,7 +3571,7 @@ namespace GameCult.Networking try { SendSchema(CultNetErrorV1Message.From( - CultNetErrorMessage.ForMessageInvalid("message", "the message could not be read"), + CultNetErrorMessage.ForUnreadableMessage(), CultNetReadableMessageId.From(payload))); } catch (Exception) diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index c572cfbf..23a7239c 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1,7 +1,9 @@ #nullable enable using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text.Json; using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; @@ -1031,6 +1033,83 @@ namespace GameCult.Networking.Tests Assert.That(rig.TextOf(KeyA), Is.Null); } + private static JsonElement Vectors() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var path = Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + return JsonDocument.Parse(File.ReadAllText(path)).RootElement.Clone(); + } + + public static IEnumerable BlankFieldCases() => + Vectors().GetProperty("blankFields").GetProperty("cases").EnumerateArray().Select(c => c.GetProperty("name").GetString()!); + + // The one fixed text of a frame that does not read or a write that names nothing, read from the vector that carries it. + private static string UnreadableVectorText() => + Vectors().GetProperty("wire").EnumerateArray() + .Single(v => v.GetProperty("name").GetString() == "error_v1_empty_schema_id_is_unreadable_and_names_the_write") + .GetProperty("message").GetProperty("error").GetString()!; + + // write-receipt-vectors.json `blankFields`: every case, on a receipt-capable and on a legacy connection. The rig's put is + // the valid document; the case's `set` replaces the named fields (the vectors' payload bytes stand for any non-empty payload). + [Test] + public async Task APutThatNamesNothingIsRefusedAsUnreadableAndANamedOneIsAdmitted( + [ValueSource(nameof(BlankFieldCases))] string name, [Values] bool capable) + { + var vectors = Vectors().GetProperty("blankFields").GetProperty("cases").EnumerateArray() + .Single(c => c.GetProperty("name").GetString() == name); + var rig = Build(); + var wire = new Wire(); + if (capable) await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + var message = rig.PutOf("m1", KeyA, "one"); + var document = message.Document!; + foreach (var field in vectors.GetProperty("set").EnumerateObject()) + { + var value = field.Value; + var text = value.ValueKind == JsonValueKind.String ? value.GetString() : null; + switch (field.Name) + { + case "schemaId": document.SchemaId = text!; break; + case "recordKey": document.RecordKey = text!; break; + case "storedAt": document.StoredAt = text!; break; + case "payloadHex": document.Payload = Convert.FromHexString(text!); break; + case "sourceRuntimeId": document.SourceRuntimeId = text; break; + case "sourceAgentId": document.SourceAgentId = text; break; + case "sourceRole": document.SourceRole = text; break; + case "tags": document.Tags = value.ValueKind == JsonValueKind.Null ? null : value.EnumerateArray().Select(t => t.GetString()!).ToArray(); break; + default: Assert.Fail($"unknown field {field.Name}"); break; + } + } + + await rig.PutHandler(message, wire.Peer); + + if (vectors.GetProperty("outcome").GetString() == "admitted") + { + if (capable) AssertAdmitted(wire.Sent.Single(), "m1", "put", document.RecordKey); + else Assert.That(wire.Sent, Is.Empty, "a legacy connection is sent no receipt"); + Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); + return; + } + + var refused = wire.Sent.Single(); + var error = capable ? AssertRefused(refused, "m1", "message_invalid").ToV0() : (CultNetErrorMessage)refused; + Assert.That(error.Code, Is.EqualTo("message_invalid")); + Assert.That(error.Details!.Field, Is.EqualTo("message")); + Assert.That(error.Error, Is.EqualTo(UnreadableVectorText())); + Assert.That(rig.TextOf(KeyA), Is.Null, "a write that names nothing is never applied"); + } + + // The fixed text is also what an undecodable frame is answered with, from the transport, not a decoder's wording. + [Test] + public async Task AnUnreadableSchemaFrameIsAnsweredWithTheFixedText() + { + var (_, wire) = await Capable(); + wire.Transport.AnswerUnreadableSchemaFrame(new byte[] { 0xC1 }); + Assert.That(AssertRefused(wire.Sent.Single(), null, "message_invalid").Error, Is.EqualTo(UnreadableVectorText())); + } + [TestCase("", "k")] [TestCase(" ", "k")] [TestCase("note.v0", "")] From 98989f90ea45c6df9a2434055fb1a3abb6f627d7 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:57:38 +0000 Subject: [PATCH 070/117] C# tests: a nil payload names nothing (mutant survivor) Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetWriteReceiptServerTests.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 23a7239c..1ccd6c20 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1101,6 +1101,25 @@ namespace GameCult.Networking.Tests Assert.That(rig.TextOf(KeyA), Is.Null, "a write that names nothing is never applied"); } + // A nil payload names no document either; it is refused like an empty one, never handed to the store. + [Test] + public async Task APutWithANilPayloadIsRefusedAsUnreadable([Values] bool capable) + { + var rig = Build(); + var wire = new Wire(); + if (capable) await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + var message = rig.PutOf("m1", KeyA, "one"); + message.Document!.Payload = null!; + + await rig.PutHandler(message, wire.Peer); + + var refused = wire.Sent.Single(); + var error = capable ? AssertRefused(refused, "m1", "message_invalid").ToV0() : (CultNetErrorMessage)refused; + Assert.That(error.Error, Is.EqualTo(UnreadableVectorText())); + Assert.That(rig.TextOf(KeyA), Is.Null); + } + // The fixed text is also what an undecodable frame is answered with, from the transport, not a decoder's wording. [Test] public async Task AnUnreadableSchemaFrameIsAnsweredWithTheFixedText() From ccbe392b2870db5570ac821fbd125c46fbe7a26e Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:02:10 +0000 Subject: [PATCH 071/117] write-receipt vectors and tests: blank is Unicode White_Space, every code point checked Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 18 +++++- .../cultmesh-rs/tests/rudp_document_server.rs | 60 ++++++++++++++++-- .../CultNetWriteReceiptServerTests.cs | 63 ++++++++++++++++++- 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 1dc39fd8..1d83ea89 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -111,8 +111,11 @@ } ], "blankFields": { - "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or only whitespace. Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.", + "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or made only of Unicode White_Space code points (the list in whiteSpaceCodePoints, the PropList property, the same in every runtime; U+200B, U+FEFF and U+180E are not White_Space). Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.", "validPutPayloadHex": "81A16101", + "whiteSpaceCodePoints": ["0009","000A","000B","000C","000D","0020","0085","00A0","1680","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","200A","2028","2029","202F","205F","3000"], + "notWhiteSpaceCodePoints": ["200B","200C","200D","2060","FEFF","180E"], + "codePointCases": "For every code point in whiteSpaceCodePoints, each of schemaId, recordKey, storedAt, sourceRuntimeId, sourceAgentId, sourceRole and one tags entry, set alone to a string of just that code point (and to that code point repeated three times), is refused; for every code point in notWhiteSpaceCodePoints the same string is not blank and the put is admitted.", "cases": [ { "name": "valid_document_is_admitted", "set": {}, "outcome": "admitted" }, { "name": "empty_schema_id", "set": { "schemaId": "" }, "outcome": "refused" }, @@ -126,6 +129,19 @@ { "name": "blank_source_agent_id", "set": { "sourceAgentId": " " }, "outcome": "refused" }, { "name": "blank_source_role", "set": { "sourceRole": "" }, "outcome": "refused" }, { "name": "blank_tag", "set": { "tags": ["a", " "] }, "outcome": "refused" }, + { "name": "nbsp_schema_id", "set": { "schemaId": "\u00a0" }, "outcome": "refused" }, + { "name": "em_space_record_key", "set": { "recordKey": "\u2003\u2003" }, "outcome": "refused" }, + { "name": "next_line_stored_at", "set": { "storedAt": "\u0085" }, "outcome": "refused" }, + { "name": "line_separator_source_runtime_id", "set": { "sourceRuntimeId": "\u2028" }, "outcome": "refused" }, + { "name": "ideographic_space_source_agent_id", "set": { "sourceAgentId": "\u3000" }, "outcome": "refused" }, + { "name": "nbsp_source_role", "set": { "sourceRole": " \u00a0 " }, "outcome": "refused" }, + { "name": "em_space_tag", "set": { "tags": ["a", "\u2003"] }, "outcome": "refused" }, + { "name": "next_line_tag", "set": { "tags": ["\u0085"] }, "outcome": "refused" }, + { "name": "zero_width_space_schema_id_is_a_name", "set": { "schemaId": "\u200b" }, "outcome": "admitted" }, + { "name": "zero_width_space_record_key_is_a_name", "set": { "recordKey": "\u200b" }, "outcome": "admitted" }, + { "name": "zero_width_space_tag_is_a_name", "set": { "tags": ["\u200b"] }, "outcome": "admitted" }, + { "name": "bom_stored_at_is_a_name", "set": { "storedAt": "\ufeff" }, "outcome": "admitted" }, + { "name": "mongolian_vowel_separator_source_role_is_a_name", "set": { "sourceRole": "\u180e" }, "outcome": "admitted" }, { "name": "source_ids_and_tags_present_and_named", "set": { "sourceRuntimeId": "r", "sourceAgentId": "a", "sourceRole": "w", "tags": ["a"] }, "outcome": "admitted" }, { "name": "source_ids_and_tags_nil", "set": { "sourceRuntimeId": null, "sourceAgentId": null, "sourceRole": null, "tags": null }, "outcome": "admitted" }, { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" } diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index b8772eaf..be667e72 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3139,11 +3139,15 @@ fn unreadable_vector_text(vectors: &serde_json::Value) -> String { /// The put frame of a `blankFields` case: the valid document with the case's `set` applied. Serialized without the /// typed encoder, because that refuses what the case is about. fn blank_field_put(vectors: &serde_json::Value, set: &serde_json::Value) -> Vec { + blank_field_put_with_id(vectors, set, "m1") +} + +fn blank_field_put_with_id(vectors: &serde_json::Value, set: &serde_json::Value, message_id: &str) -> Vec { #[derive(serde::Serialize)] #[serde(rename_all = "camelCase")] struct Put { schema_version: &'static str, - message_id: &'static str, + message_id: String, document: CultNetRawDocumentRecord, } let hex = set.get("payloadHex").and_then(|v| v.as_str()).unwrap_or(vectors["blankFields"]["validPutPayloadHex"].as_str().unwrap()); @@ -3160,7 +3164,7 @@ fn blank_field_put(vectors: &serde_json::Value, set: &serde_json::Value) -> Vec< source_role: optional("sourceRole"), tags: set.get("tags").and_then(|v| v.as_array()).map(|tags| tags.iter().map(|t| t.as_str().unwrap().to_string()).collect()), }; - rmp_serde::to_vec_named(&Put { schema_version: "cultnet.document_put_raw.v0", message_id: "m1", document }).unwrap() + rmp_serde::to_vec_named(&Put { schema_version: "cultnet.document_put_raw.v0", message_id: message_id.to_string(), document }).unwrap() } /// Each `blankFields` case: a receipt-capable connection is answered with the admitted receipt, or with the one fixed @@ -3188,8 +3192,8 @@ fn a_put_that_names_nothing_is_refused_as_unreadable_naming_the_write_and_a_name vec![CultNetMessage::WriteAdmitted { message_id: "m1".into(), operation: "put".into(), - schema_id: "note.v0".into(), - record_key: "note:1".into(), + schema_id: case["set"]["schemaId"].as_str().unwrap_or("note.v0").into(), + record_key: case["set"]["recordKey"].as_str().unwrap_or("note:1").into(), }], "{name}" ); @@ -3207,6 +3211,54 @@ fn a_put_that_names_nothing_is_refused_as_unreadable_naming_the_write_and_a_name Ok(()) } +fn code_point_string(hex: &str, times: usize) -> String { + std::char::from_u32(u32::from_str_radix(hex, 16).unwrap()).unwrap().to_string().repeat(times) +} + +/// The Unicode White_Space set is the one blank rule: every listed code point, alone or repeated, blanks every field +/// the rule covers; U+200B, U+FEFF, U+180E and the other listed non-White_Space code points name something. +#[test] +fn blank_means_exactly_the_unicode_white_space_code_points_in_every_field() -> Result<()> { + let vectors = vectors(); + let fixed = unreadable_vector_text(&vectors); + let fields = ["schemaId", "recordKey", "storedAt", "sourceRuntimeId", "sourceAgentId", "sourceRole", "tags"]; + let space = vectors["blankFields"]["whiteSpaceCodePoints"].as_array().unwrap(); + let not_space = vectors["blankFields"]["notWhiteSpaceCodePoints"].as_array().unwrap(); + assert_eq!(space.len(), 25); + for (index, (hex, blank)) in space.iter().map(|h| (h, true)).chain(not_space.iter().map(|h| (h, false))).enumerate() { + let hex = hex.as_str().unwrap(); + for (field_index, field) in fields.iter().enumerate() { + // A session that is sent refusals is allowed to end, so each code point and field gets its own. + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 600 + (index * fields.len() + field_index) as u32)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![server_hello()]); + let mut admitted = 0; + for times in [1, 3] { + let text = code_point_string(hex, times); + let value = if *field == "tags" { serde_json::json!(["a", text]) } else { serde_json::json!(text) }; + let set = serde_json::json!({ *field: value }); + let id = format!("{hex}-{field}-{times}"); + peer.send_reliable("schema", blank_field_put_with_id(&vectors, &set, &id))?; + let answers = frames_after(&mut server, &mut peer, 1)?; + let what = format!("U+{hex} in {field} x{times}"); + assert!(!answers.is_empty(), "{what}: no answer"); + if blank { + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some(id)), "{what}"); + assert_eq!(error_text_and_field(&answers[0]), (fixed.clone(), Some("message".into())), "{what}"); + } else { + assert!(matches!(answers[0], CultNetMessage::WriteAdmitted { .. }), "{what} is a name: {:?}", answers[0]); + admitted += 1; + } + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), admitted, "U+{hex} in {field}: only named writes are applied"); + } + } + Ok(()) +} + /// The same cases on a connection that never opted in: the same refusal as `error.v0`, the same text, nothing applied. #[test] fn a_put_that_names_nothing_is_refused_the_same_way_on_a_legacy_connection() -> Result<()> { diff --git a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs index 1ccd6c20..e42961a2 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs @@ -1089,7 +1089,8 @@ namespace GameCult.Networking.Tests { if (capable) AssertAdmitted(wire.Sent.Single(), "m1", "put", document.RecordKey); else Assert.That(wire.Sent, Is.Empty, "a legacy connection is sent no receipt"); - Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); + if (!vectors.GetProperty("set").EnumerateObject().Any(f => f.Name is "schemaId" or "recordKey")) + Assert.That(rig.TextOf(KeyA), Is.EqualTo("one")); return; } @@ -1101,6 +1102,66 @@ namespace GameCult.Networking.Tests Assert.That(rig.TextOf(KeyA), Is.Null, "a write that names nothing is never applied"); } + public static IEnumerable WhiteSpaceCodePoints() => + Vectors().GetProperty("blankFields").GetProperty("whiteSpaceCodePoints").EnumerateArray().Select(c => c.GetString()!); + + public static IEnumerable NonWhiteSpaceCodePoints() => + Vectors().GetProperty("blankFields").GetProperty("notWhiteSpaceCodePoints").EnumerateArray().Select(c => c.GetString()!); + + private static readonly string[] BlankRuleFields = + { "schemaId", "recordKey", "storedAt", "sourceRuntimeId", "sourceAgentId", "sourceRole", "tags" }; + + // Blank is the Unicode White_Space property in every runtime (write-receipt.md, Required fields), not whatever the + // platform's char.IsWhiteSpace says: each listed code point, alone or repeated, in each covered field is refused as + // unreadable and never applied; each listed non-White_Space code point (U+200B, U+FEFF, U+180E, ...) is a name. + private async Task AssertBlankRule(string hex, bool blank) + { + foreach (var field in BlankRuleFields) + foreach (var times in new[] { 1, 3 }) + { + var text = string.Concat(Enumerable.Repeat(char.ConvertFromUtf32(Convert.ToInt32(hex, 16)), times)); + var rig = Build(); + var wire = new Wire(); + await rig.Hello(wire, BothFrames); + wire.Sent.Clear(); + var message = rig.PutOf("m1", KeyA, "one"); + var document = message.Document!; + switch (field) + { + case "schemaId": document.SchemaId = text; break; + case "recordKey": document.RecordKey = text; break; + case "storedAt": document.StoredAt = text; break; + case "sourceRuntimeId": document.SourceRuntimeId = text; break; + case "sourceAgentId": document.SourceAgentId = text; break; + case "sourceRole": document.SourceRole = text; break; + default: document.Tags = new[] { "a", text }; break; + } + var what = $"U+{hex} in {field} x{times}"; + + await rig.PutHandler(message, wire.Peer); + + if (blank) + { + var error = AssertRefused(wire.Sent.Single(), "m1", "message_invalid").ToV0(); + Assert.That(error.Details!.Field, Is.EqualTo("message"), what); + Assert.That(error.Error, Is.EqualTo(UnreadableVectorText()), what); + Assert.That(rig.TextOf(KeyA), Is.Null, $"{what}: never applied"); + } + else + { + AssertAdmitted(wire.Sent.Single(), "m1", "put", document.RecordKey); + } + } + } + + [Test] + public async Task EveryUnicodeWhiteSpaceCodePointIsBlank([ValueSource(nameof(WhiteSpaceCodePoints))] string hex) => + await AssertBlankRule(hex, blank: true); + + [Test] + public async Task CodePointsOutsideWhiteSpaceNameSomething([ValueSource(nameof(NonWhiteSpaceCodePoints))] string hex) => + await AssertBlankRule(hex, blank: false); + // A nil payload names no document either; it is refused like an empty one, never handed to the store. [Test] public async Task APutWithANilPayloadIsRefusedAsUnreadable([Values] bool capable) From db5e8834f38846f41107281339f5d8a011ef89ad Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:02:49 +0000 Subject: [PATCH 072/117] write-receipt: blank is exactly Unicode White_Space; C# server checks the listed set in one place Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- src/GameCult.Networking/CultNetDatabaseServer.cs | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 35756ecf..a3ad68dc 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -36,7 +36,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is empty or only whitespace; on a put's document, also a `storedAt` that is empty or only whitespace, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and empty or only whitespace, and a `tags` entry that is empty or only whitespace. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. + - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF and U+180E are not White_Space: a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index 31822ade..e1694131 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -394,18 +394,26 @@ namespace GameCult.Networking // (write-receipt.md, "Required fields"), never handed to the store: a blank schemaId, recordKey or storedAt, an empty // payload, and a source id or tag that is present but blank. private static bool IsUnreadableWriteTarget(string? schemaId, string? recordKey) => - string.IsNullOrWhiteSpace(schemaId) || string.IsNullOrWhiteSpace(recordKey); + IsBlank(schemaId) || IsBlank(recordKey); private static bool IsUnreadableDocument(CultNetRawDocumentRecord document) => IsUnreadableWriteTarget(document.SchemaId, document.RecordKey) - || string.IsNullOrWhiteSpace(document.StoredAt) + || IsBlank(document.StoredAt) || document.Payload == null || document.Payload.Length == 0 || IsPresentButBlank(document.SourceRuntimeId) || IsPresentButBlank(document.SourceAgentId) || IsPresentButBlank(document.SourceRole) - || document.Tags != null && document.Tags.Any(tag => string.IsNullOrWhiteSpace(tag)); + || document.Tags != null && document.Tags.Any(IsBlank); - private static bool IsPresentButBlank(string? value) => value != null && string.IsNullOrWhiteSpace(value); + private static bool IsPresentButBlank(string? value) => value != null && IsBlank(value); + + // Blank is the Unicode White_Space property, listed here and in write-receipt.md, not char.IsWhiteSpace: the platform's + // own test follows its Unicode tables (and has put U+180E in the set), the spec's does not move. A null is not text. + private static bool IsBlank(string? value) => value == null || value.All(IsWhiteSpaceCodePoint); + + private static bool IsWhiteSpaceCodePoint(char c) => + c is (>= '\u0009' and <= '\u000D') or '\u0020' or '\u0085' or '\u00A0' or '\u1680' + or (>= '\u2000' and <= '\u200A') or '\u2028' or '\u2029' or '\u202F' or '\u205F' or '\u3000'; // A refusal: error.v1 carrying the write's id for a receipt-capable connection, the v0 error for any other. private static void Refuse(CultNetServerPeer peer, CultNetReceiptedWrite? write, CultNetErrorMessage error) From cbb336f501ac781afa83bf87c6ece79390377897 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:03:27 +0000 Subject: [PATCH 073/117] cultmesh-kotlin: decode write_admitted.v0 and error.v1, content hash, WriteTracker, vector replay Strict decode of the two receipt frames (and error.v0), hello capability test, the dedupe content hash and the client settle rules, ported from cultnet-ts. write-receipt-vectors.json replays 38/38 (constants, contentHash, wire, schemaInvalid, scenarios, v0 limit_reached); dedupe and blankFields are server inputs and not run. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-kotlin/README.md | 11 + packages/cultmesh-kotlin/build.ps1 | 3 + .../kotlin/org/gamecult/cultmesh/CultMesh.kt | 1 + .../org/gamecult/cultmesh/WriteReceipt.kt | 232 ++++++++++++++++++ .../gamecult/cultmesh/WriteReceiptVectors.kt | 224 +++++++++++++++++ 5 files changed, 471 insertions(+) create mode 100644 packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt create mode 100644 packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt diff --git a/packages/cultmesh-kotlin/README.md b/packages/cultmesh-kotlin/README.md index 3da68679..ff69c346 100644 --- a/packages/cultmesh-kotlin/README.md +++ b/packages/cultmesh-kotlin/README.md @@ -474,3 +474,14 @@ powershell -ExecutionPolicy Bypass -File .\packages\cultmesh-kotlin\build.ps1 The build writes `artifacts/cultmesh-kotlin/cultmesh-kotlin.jar`. It also runs the built-in RUDP packet fixture and localhost UDP socket self-test. + +## Write Receipt + +`WriteReceipt.kt` decodes `cultnet.write_admitted.v0` and `cultnet.error.v1` (and `error.v0`) strictly against their +closed schemas (`decodeWriteAdmitted`, `decodeErrorFrame`), computes the dedupe `writeContentHash`, tests a hello with +`helloAdvertisesWriteReceipts`, and settles sent writes as admitted, refused or unconfirmed in `WriteTracker` +(pure state machine, abstract clock; the caller owns transport and timers). Kotlin has no write client yet, so nothing +sends a put and waits on this. `WriteReceiptVectors.kt` replays +`contracts/cultnet/interop/write-receipt-vectors.json` (constants, contentHash, wire, schemaInvalid, scenarios; the +server-side `dedupe` and `blankFields` sections are not run); `build.ps1` runs it after the self-test, or run +`CultMeshKt write-receipt-vectors `. diff --git a/packages/cultmesh-kotlin/build.ps1 b/packages/cultmesh-kotlin/build.ps1 index aabacbcd..42c22358 100644 --- a/packages/cultmesh-kotlin/build.ps1 +++ b/packages/cultmesh-kotlin/build.ps1 @@ -26,4 +26,7 @@ $sources = Get-ChildItem $src -Recurse -Filter *.kt | Select-Object -ExpandPrope if ($LASTEXITCODE -ne 0) { throw "kotlinc failed with exit code $LASTEXITCODE" } & $java -cp "$jarPath;$kotlinStdlib" org.gamecult.cultmesh.CultMeshKt if ($LASTEXITCODE -ne 0) { throw "cultmesh-kotlin self-test failed with exit code $LASTEXITCODE" } +$vectors = Join-Path $repoRoot "contracts\cultnet\interop\write-receipt-vectors.json" +& $java -cp "$jarPath;$kotlinStdlib" org.gamecult.cultmesh.CultMeshKt write-receipt-vectors $vectors +if ($LASTEXITCODE -ne 0) { throw "write-receipt vector replay failed with exit code $LASTEXITCODE" } Write-Host "Built $jarPath" diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 7d052785..d9cdfad7 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -3424,6 +3424,7 @@ fun main(args: Array) { "rudp-dial-message-once" -> rudpDialSchemaMessageOnce(options) "serve" -> interopServe(options) "dial" -> interopDial(options) + "write-receipt-vectors" -> if (!replayWriteReceiptVectors(args.getOrNull(1) ?: error("usage: write-receipt-vectors "))) kotlin.system.exitProcess(1) else -> error("Unknown mode ${args[0]}") } } diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt new file mode 100644 index 00000000..4060f08a --- /dev/null +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -0,0 +1,232 @@ +package org.gamecult.cultmesh + +// The write receipt (contracts/cultnet/interop/write-receipt.md): the two frames, their strict decode, the dedupe +// content hash, the hello capability test and the client settle rules. Mirrors packages/cultnet-ts (contracts.ts, +// write-tracker.ts) and packages/cultnet-rs (write_content_hash). The replay of write-receipt-vectors.json lives in +// WriteReceiptVectors.kt. Server rules (dedupe, limits, blank fields) are not a decoder's business and are not here. + +import java.io.IOException +import java.security.MessageDigest + +const val WRITE_ADMITTED_VERSION = "cultnet.write_admitted.v0" +const val ERROR_V1_VERSION = "cultnet.error.v1" +const val ERROR_V0_VERSION = "cultnet.error.v0" + +/** A connection is receipt-capable only when its hello lists both frames. */ +val CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS: List = listOf(WRITE_ADMITTED_VERSION, ERROR_V1_VERSION) + +const val RECEIPT_RESULT_WINDOW_MS = 5000L +const val LEGACY_RESULT_WINDOW_MS = 500L + +fun resultWindowMs(serverAdvertisesReceipts: Boolean): Long = + if (serverAdvertisesReceipts) RECEIPT_RESULT_WINDOW_MS else LEGACY_RESULT_WINDOW_MS + +/** The closed refusal-code set of error.v0; error.v1 adds `limit_reached`. */ +val CULTNET_ERROR_V0_CODES: Set = setOf( + "selection_invalid", "cursor_stale", "cursor_invalid", "reference_outside_target", "variant_unsupported", + "unowned_schema", "write_not_authorized", "unsupported_schema_version", "message_invalid", +) +val CULTNET_ERROR_V1_CODES: Set = CULTNET_ERROR_V0_CODES + "limit_reached" + +fun helloAdvertisesWriteReceipts(hello: CultNetMessage): Boolean { + val versions = hello.body["supportedMessageVersions"] as? List<*> ?: return false + return CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS.all { it in versions } +} + +data class CultNetWriteAdmitted(val messageId: String, val operation: String, val schemaId: String, val recordKey: String) { + fun toMessage() = CultNetMessage( + WRITE_ADMITTED_VERSION, + linkedMapOf("messageId" to messageId, "operation" to operation, "schemaId" to schemaId, "recordKey" to recordKey), + ) +} + +data class CultNetErrorDetails(val field: String?, val value: String?, val asOf: Long?, val current: Long?) + +/** error.v0 (`messageId` always null, `version` 0) or error.v1 (`version` 1). */ +data class CultNetErrorFrame( + val version: Int, + val error: String, + val routingHint: Map?, + val code: String?, + val details: CultNetErrorDetails?, + val messageId: String?, +) { + fun toMessage(): CultNetMessage { + val body = linkedMapOf( + "error" to error, + "routingHint" to routingHint, + "code" to code, + "details" to details?.let { + linkedMapOf("field" to it.field, "value" to it.value, "asOf" to it.asOf, "current" to it.current) + }, + ) + if (version == 1) body["messageId"] = messageId + return CultNetMessage(if (version == 1) ERROR_V1_VERSION else ERROR_V0_VERSION, body) + } +} + +/** + * Strict decode of a write_admitted.v0 message against its closed schema: exactly five keys, a non-empty string each, + * operation put|delete. + */ +fun decodeWriteAdmitted(message: CultNetMessage): CultNetWriteAdmitted { + if (message.schemaVersion != WRITE_ADMITTED_VERSION) throw IOException("not a $WRITE_ADMITTED_VERSION message") + closedKeys(message, setOf("messageId", "operation", "schemaId", "recordKey")) + val operation = nonEmpty(message, "operation") + if (operation != "put" && operation != "delete") throw IOException("operation must be put or delete") + return CultNetWriteAdmitted(nonEmpty(message, "messageId"), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) +} + +/** Strict decode of error.v0 (no messageId, no limit_reached) or error.v1 (messageId required, string or nil, never empty). */ +fun decodeErrorFrame(message: CultNetMessage): CultNetErrorFrame { + val version = when (message.schemaVersion) { + ERROR_V1_VERSION -> 1 + ERROR_V0_VERSION -> 0 + else -> throw IOException("not an error message") + } + closedKeys(message, setOf("error", "routingHint", "code", "details") + if (version == 1) setOf("messageId") else emptySet()) + val error = nonEmpty(message, "error") + val codeValue = message.body["code"] + if (codeValue != null && (codeValue !is String || codeValue !in (if (version == 1) CULTNET_ERROR_V1_CODES else CULTNET_ERROR_V0_CODES))) { + throw IOException("unknown error code") + } + var messageId: String? = null + if (version == 1) { + val id = message.body["messageId"] + if (id != null && (id !is String || id.isEmpty())) throw IOException("messageId must be a non-empty string or nil") + messageId = id as String? + } + return CultNetErrorFrame(version, error, routingHintOf(message.body["routingHint"]), codeValue as String?, detailsOf(message.body["details"]), messageId) +} + +private fun closedKeys(message: CultNetMessage, required: Set) { + if (!message.body.keys.containsAll(required)) throw IOException("${message.schemaVersion} is missing ${required - message.body.keys}") + val unknown = message.body.keys - required + if (unknown.isNotEmpty()) throw IOException("${message.schemaVersion} carries unknown keys $unknown") +} + +private fun nonEmpty(message: CultNetMessage, key: String): String { + val value = message.body[key] + if (value !is String || value.isEmpty()) throw IOException("$key must be a non-empty string") + return value +} + +private fun routingHintOf(value: Any?): Map? { + if (value == null) return null + if (value !is Map<*, *> || value.keys != setOf("shard", "reason")) throw IOException("routingHint must be nil or {shard, reason}") + if (value["shard"] != null && value["shard"] !is Map<*, *>) throw IOException("routingHint.shard must be an object or nil") + if (value["reason"] !is String) throw IOException("routingHint.reason must be a string") + @Suppress("UNCHECKED_CAST") + return value as Map +} + +private fun detailsOf(value: Any?): CultNetErrorDetails? { + if (value == null) return null + if (value !is Map<*, *> || value.keys != setOf("field", "value", "asOf", "current")) { + throw IOException("details must be nil or {field, value, asOf, current}") + } + fun text(key: String): String? = value[key].let { if (it == null || it is String) it as String? else throw IOException("details.$key must be a string or nil") } + fun cursor(key: String): Long? = value[key].let { + if (it == null) null else if (it is Long && it >= 0) it else throw IOException("details.$key must be a non-negative integer or nil") + } + return CultNetErrorDetails(text("field"), text("value"), cursor("asOf"), cursor("current")) +} + +/** sha256 of the MessagePack array [operation, schemaId, recordKey, payload]: payload a bin for a put, nil for a delete. Lowercase hex. */ +fun writeContentHash(operation: String, schemaId: String, recordKey: String, payload: ByteArray?): String { + val writer = MessagePackWriter().array(4).string(operation).string(schemaId).string(recordKey) + if (payload == null) writer.nil() else writer.binary(payload) + return MessageDigest.getInstance("SHA-256").digest(writer.toByteArray()).joinToString("") { "%02x".format(it) } +} + +/** Why nothing settled a write. */ +enum class WriteUnconfirmedReason(val wire: String) { WINDOW_ELAPSED("window_elapsed"), SESSION_ENDED("session_ended"), LOCAL_SEND_FAILED("local_send_failed") } + +/** What a late answer is recorded as on a write that already settled unconfirmed. */ +enum class WriteRecorded(val wire: String) { ADMITTED_LATE("admitted_late"), REFUSED_LATE("refused_late") } + +/** What the caller was told. Settled once; never changed by a later frame. */ +sealed class WriteSettlement { + object Admitted : WriteSettlement() + data class Refused(val code: String?, val error: String, val response: CultNetErrorFrame) : WriteSettlement() + data class Unconfirmed(val reason: WriteUnconfirmedReason) : WriteSettlement() +} + +/** + * The client half of the receipt rules: a pure state machine with an abstract clock, one decision, how each sent write + * settles. Transport, timers and decoding belong to the caller; `frame` takes a message and decodes it strictly. + */ +class WriteTracker(private val windowMs: Long, private val onSettle: (String, WriteSettlement) -> Unit = { _, _ -> }) { + private class Entry(val sentAtMs: Long) { + var settlement: WriteSettlement? = null + var recorded: WriteRecorded? = null + } + + private val writes = linkedMapOf() + + /** Registers a write sent at [atMs]. Re-sending a known id is the same write and does not restart its window. */ + fun send(messageId: String, atMs: Long) { + tick(atMs) + writes.getOrPut(messageId) { Entry(atMs) } + } + + /** Feeds one received message. Only write_admitted, error.v1 and error.v0 answer; anything else is ignored. */ + fun frame(message: CultNetMessage, atMs: Long) { + tick(atMs) + when (message.schemaVersion) { + WRITE_ADMITTED_VERSION -> { + answer(decodeWriteAdmitted(message).messageId, WriteSettlement.Admitted, WriteRecorded.ADMITTED_LATE) + return + } + ERROR_V1_VERSION, ERROR_V0_VERSION -> { + val error = decodeErrorFrame(message) + val refusal = WriteSettlement.Refused(error.code, error.error, error) + if (error.messageId != null) { + answer(error.messageId, refusal, WriteRecorded.REFUSED_LATE) + return + } + // An error that names no write is attributable only when exactly one write is unanswered. + val unanswered = writes.filterValues { it.settlement == null } + if (unanswered.size == 1) settle(unanswered.keys.first(), unanswered.values.first(), refusal) + } + else -> return + } + } + + /** Every unanswered write whose window, counted from its own send, has run settles unconfirmed. */ + fun tick(atMs: Long) { + for ((id, entry) in writes) { + if (entry.settlement == null && atMs - entry.sentAtMs >= windowMs) settle(id, entry, WriteSettlement.Unconfirmed(WriteUnconfirmedReason.WINDOW_ELAPSED)) + } + } + + fun sessionEnd(atMs: Long) { + tick(atMs) + for ((id, entry) in writes) if (entry.settlement == null) settle(id, entry, WriteSettlement.Unconfirmed(WriteUnconfirmedReason.SESSION_ENDED)) + } + + /** This client's own send of [messageId] failed: that write settles unconfirmed; the others are untouched. */ + fun sendFailed(messageId: String, atMs: Long) { + val entry = writes[messageId] + if (entry != null && entry.settlement == null) settle(messageId, entry, WriteSettlement.Unconfirmed(WriteUnconfirmedReason.LOCAL_SEND_FAILED)) + tick(atMs) + } + + /** The settlement told to the caller, or null while unanswered or unknown. */ + fun settlement(messageId: String): WriteSettlement? = writes[messageId]?.settlement + + /** The late answer recorded on an unconfirmed write. */ + fun recorded(messageId: String): WriteRecorded? = writes[messageId]?.recorded + + private fun answer(messageId: String, settlement: WriteSettlement, late: WriteRecorded) { + val entry = writes[messageId] ?: return // names a write that is not in flight: settles nothing + val current = entry.settlement + if (current == null) settle(messageId, entry, settlement) + else if (current is WriteSettlement.Unconfirmed && entry.recorded == null) entry.recorded = late + } + + private fun settle(messageId: String, entry: Entry, settlement: WriteSettlement) { + entry.settlement = settlement + onSettle(messageId, settlement) + } +} diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt new file mode 100644 index 00000000..033ef8fd --- /dev/null +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -0,0 +1,224 @@ +package org.gamecult.cultmesh + +// Replays contracts/cultnet/interop/write-receipt-vectors.json against this runtime: constants, contentHash, every +// `wire` frame (decode to the vector's fields, re-encode to its bytes), every `schemaInvalid` frame (must be refused), +// and every `scenarios` event replay through WriteTracker. `dedupe` and `blankFields` are server inputs and are not +// run here. Mode: `java -cp ... org.gamecult.cultmesh.CultMeshKt write-receipt-vectors `. + +import java.io.File +import java.io.IOException + +fun replayWriteReceiptVectors(path: String): Boolean { + @Suppress("UNCHECKED_CAST") + val vectors = JsonReader(File(path).readText(Charsets.UTF_8)).readDocument() as Map + val results = linkedMapOf>() + fun check(name: String, run: () -> List) { + results[name] = try { run() } catch (e: Throwable) { listOf("threw: $e") } + } + + val constants = obj(vectors["constants"]) + check("constants") { + listOfNotNull( + "RECEIPT window".takeIf { constants["RECEIPT_RESULT_WINDOW_MS"] != RECEIPT_RESULT_WINDOW_MS }, + "LEGACY window".takeIf { constants["LEGACY_RESULT_WINDOW_MS"] != LEGACY_RESULT_WINDOW_MS }, + ) + } + + for (case in list(obj(vectors["contentHash"])["cases"])) { + val c = obj(case) + check("contentHash/${c["name"]}") { + val got = writeContentHash(c["operation"] as String, c["schemaId"] as String, c["recordKey"] as String, (c["payloadHex"] as String?)?.let(::unhex)) + if (got == c["sha256"]) emptyList() else listOf("$got != ${c["sha256"]}") + } + } + + for (case in list(vectors["wire"])) { + val c = obj(case) + check("wire/${c["name"]}") { + val bytes = unhex(c["messagePackHex"] as String) + val problems = mutableListOf() + val message = decodeReceiptFrame(parseCultNetMessage(bytes)) + if (message.toWireMap() != c["message"]) problems += "decoded ${message.toWireMap()} != ${c["message"]}" + val again = message.toBytes().joinToString("") { "%02X".format(it) } + if (again != (c["messagePackHex"] as String).uppercase()) problems += "re-encoded bytes $again != captured ${c["messagePackHex"]}" + problems + } + } + + for (case in list(vectors["schemaInvalid"])) { + val c = obj(case) + check("schemaInvalid/${c["name"]}") { + try { + decodeReceiptFrame(parseCultNetMessage(unhex(c["messagePackHex"] as String))) + listOf("accepted a frame that must be refused (${c["why"]})") + } catch (_: IOException) { + emptyList() + } + } + } + + // Not a vector: `limit_reached` is in error.v1's enum only (write-receipt.md), as the TS and Rust suites also pin. + check("error_v0_refuses_the_v1_only_limit_reached_code") { + val body = linkedMapOf("error" to "x", "routingHint" to null, "code" to "limit_reached", "details" to null) + val v1 = decodeErrorFrame(CultNetMessage(ERROR_V1_VERSION, body + ("messageId" to null))).code + try { + decodeErrorFrame(CultNetMessage(ERROR_V0_VERSION, body)) + listOf("error.v0 accepted limit_reached") + } catch (_: IOException) { + if (v1 == "limit_reached") emptyList() else listOf("error.v1 lost limit_reached") + } + } + + for (case in list(obj(vectors["scenarios"])["cases"])) { + val scenario = obj(case) + check("scenario/${scenario["name"]}") { + val problems = mutableListOf() + // The fake server states its capability the only way a real one does: in its hello. + val hello = cultNetHello( + "fake-server", "fake", + supportedMessageVersions = if (scenario["server"] == "receipt") CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS else emptyList(), + ) + val windowMs = resultWindowMs(helloAdvertisesWriteReceipts(hello)) + if (windowMs != constants[scenario["window"] as String]) problems += "window $windowMs != ${scenario["window"]}" + val told = linkedMapOf() + val tracker = WriteTracker(windowMs) { id, settlement -> + if (id in told) problems += "$id settled twice" + told[id] = settlement + } + for (event in list(scenario["events"]).map(::obj).sortedBy { it["atMs"] as Long }) { + val at = event["atMs"] as Long + when { + "send" in event -> tracker.send(event["send"] as String, at) + // A frame reaches the client as bytes: the fake server encodes it, the client decodes it. + "frame" in event -> tracker.frame(parseCultNetMessage(MessagePackWriter().value(event["frame"]).toByteArray()), at) + "tick" in event -> tracker.tick(at) + "sessionEnd" in event -> tracker.sessionEnd(at) + // transportAck: delivery is not an answer; the tracker has no such input and must not need one. + } + } + val expect = obj(scenario["expect"]) + for ((id, wantAny) in expect) { + val want = obj(wantAny) + val got = told[id] + if (got == null) { problems += "$id: never settled, wanted ${want["outcome"]}"; continue } + val (outcome, code, reason) = when (got) { + is WriteSettlement.Admitted -> Triple("admitted", null, null) + is WriteSettlement.Refused -> Triple("refused", got.code, null) + is WriteSettlement.Unconfirmed -> Triple("unconfirmed", null, got.reason.wire) + } + if (outcome != want["outcome"]) problems += "$id: told $outcome, wanted ${want["outcome"]}" + if ("code" in want && code != want["code"]) problems += "$id: code $code, wanted ${want["code"]}" + if ("reason" in want && reason != want["reason"]) problems += "$id: reason $reason, wanted ${want["reason"]}" + if (tracker.recorded(id)?.wire != want["recorded"]) problems += "$id: recorded ${tracker.recorded(id)?.wire}, wanted ${want["recorded"]}" + } + for (id in told.keys) if (id !in expect) problems += "$id: settled but the scenario expects nothing for it" + problems + } + } + + for ((name, problems) in results) { + println(if (problems.isEmpty()) "PASS $name" else "FAIL $name") + problems.forEach { println(" $it") } + } + val failed = results.values.count { it.isNotEmpty() } + println("${results.size - failed}/${results.size} passed") + return failed == 0 +} + +/** Strict decode of whichever receipt-family frame this is, returned as the message its typed form encodes; anything else is refused. */ +private fun decodeReceiptFrame(message: CultNetMessage): CultNetMessage = when (message.schemaVersion) { + WRITE_ADMITTED_VERSION -> decodeWriteAdmitted(message).toMessage() + ERROR_V1_VERSION, ERROR_V0_VERSION -> decodeErrorFrame(message).toMessage() + else -> throw IOException("not a receipt frame: ${message.schemaVersion}") +} + +@Suppress("UNCHECKED_CAST") +private fun obj(value: Any?): Map = value as Map +private fun list(value: Any?): List = value as List + +private fun unhex(text: String): ByteArray = ByteArray(text.length / 2) { text.substring(it * 2, it * 2 + 2).toInt(16).toByte() } + +/** A small JSON reader for the vector file: objects, arrays, strings, integers, true/false/null. */ +private class JsonReader(private val text: String) { + private var at = 0 + + fun readDocument(): Any? = readValue().also { skipSpace(); if (at != text.length) throw IOException("trailing JSON") } + + private fun skipSpace() { while (at < text.length && text[at].isWhitespace()) at++ } + + private fun readValue(): Any? { + skipSpace() + return when (val c = text[at]) { + '{' -> { + at++ + val map = linkedMapOf() + skipSpace() + if (text[at] == '}') { at++; return map } + do { + skipSpace() + val key = readString() + skipSpace(); expect(':') + map[key] = readValue() + skipSpace() + } while (closes('}')) + map + } + '[' -> { + at++ + val items = mutableListOf() + skipSpace() + if (text[at] == ']') { at++; return items } + do { + items += readValue() + skipSpace() + } while (closes(']')) + items + } + '"' -> readString() + 't' -> literal("true", true) + 'f' -> literal("false", false) + 'n' -> literal("null", null) + else -> { + if (c != '-' && !c.isDigit()) throw IOException("unexpected '$c' at $at") + val start = at++ + while (at < text.length && text[at].isDigit()) at++ + text.substring(start, at).toLong() + } + } + } + + /** After an element: false when [close] ends the container, true after a comma. */ + private fun closes(close: Char): Boolean { + val c = text[at++] + if (c == close) return false + if (c != ',') throw IOException("expected , or $close at ${at - 1}") + return true + } + + private fun literal(word: String, value: Any?): Any? { + if (!text.startsWith(word, at)) throw IOException("bad literal at $at") + at += word.length + return value + } + + private fun expect(c: Char) { if (text[at++] != c) throw IOException("expected $c at ${at - 1}") } + + private fun readString(): String { + expect('"') + val out = StringBuilder() + while (true) { + val c = text[at++] + when (c) { + '"' -> return out.toString() + '\\' -> when (val e = text[at++]) { + 'n' -> out.append('\n'); 't' -> out.append('\t'); 'r' -> out.append('\r') + 'b' -> out.append('\b'); 'f' -> out.append('\u000c') + 'u' -> { out.append(text.substring(at, at + 4).toInt(16).toChar()); at += 4 } + '"', '\\', '/' -> out.append(e) + else -> throw IOException("bad escape \\$e") + } + else -> out.append(c) + } + } + } +} From f6c61b1ebf2aa7723362d30cf565ab9f188952f0 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:08 +0000 Subject: [PATCH 074/117] cultnet-py: decode cultnet.write_admitted.v0 and cultnet.error.v1 against the write-receipt vectors Strict closed-schema decode, content hash helper, hello capability; vectors contentHash, wire and schemaInvalid replayed. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-py/CHANGELOG.md | 1 + .../cultnet-py/src/cultnet_py/__init__.py | 8 + .../cultnet-py/src/cultnet_py/messages.py | 90 +++++-- .../cultnet-py/tests/test_write_receipt.py | 241 ++++++++++++++++++ 4 files changed, 317 insertions(+), 23 deletions(-) create mode 100644 packages/cultnet-py/tests/test_write_receipt.py diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index b6bfa92b..5e4b0cfa 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client. - Added `message_invalid` to `CULTNET_ERROR_CODES` and `message_invalid_error(field, reason)`, byte-identical to the C# `ForMessageInvalid`. `cultmesh-py` refuses a malformed put or delete with it instead of the out-of-schema `malformed_document_put` and `malformed_document_delete`, which are removed. - Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per remote endpoint, ported from the C# `CultNetRudpSocketTransportServer` and held to its bytes by diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index bcf2fcf0..e4a22779 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -39,6 +39,8 @@ from .cultmesh_contracts import ( from .framing import decode_frame, encode_frame, read_frame, write_frame from .messages import ( CULTNET_ERROR_CODES, + CULTNET_ERROR_V1_CODES, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, CultNetMessage, CultNetWitnessArtifactBundle, compute_simulation_claim_hash, @@ -51,7 +53,9 @@ from .messages import ( error_message, message_invalid_error, unsupported_schema_version_error, + write_content_hash, hello, + hello_advertises_write_receipts, login, login_success, parse_message, @@ -176,6 +180,8 @@ __all__ = [ "verse_catalog_request", "verse_from_wire", "CULTNET_ERROR_CODES", + "CULTNET_ERROR_V1_CODES", + "CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS", "CultNetMessage", "CultNetWitnessArtifactBundle", "CultNetDatabaseSubscription", @@ -266,6 +272,8 @@ __all__ = [ "message_invalid_error", "unsupported_schema_version_error", "hello", + "hello_advertises_write_receipts", + "write_content_hash", "INTEROP_WIRE_CONTRACT", "login", "login_success", diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index 1a1f8e88..f2d89296 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -29,7 +29,11 @@ def parse_message(payload: bytes) -> CultNetMessage: body = dict(decoded) del body["schemaVersion"] if schema_version == "cultnet.error.v0": - _validate_error_body(body) + _validate_error_body(body, "cultnet.error.v0", CULTNET_ERROR_CODES) + elif schema_version == "cultnet.error.v1": + _validate_error_body(body, "cultnet.error.v1", CULTNET_ERROR_V1_CODES) + elif schema_version == "cultnet.write_admitted.v0": + _validate_write_admitted_body(body) return CultNetMessage(schema_version=schema_version, body=body) @@ -47,6 +51,13 @@ CULTNET_ERROR_CODES: tuple[str, ...] = ( "message_invalid", ) +# cultnet.error.v1 (contracts/cultnet/interop/cultnet.error.v1.schema.json): v0's codes plus limit_reached. +CULTNET_ERROR_V1_CODES: tuple[str, ...] = (*CULTNET_ERROR_CODES, "limit_reached") + +# The two frames a receipt-capable hello lists in supportedMessageVersions; a peer lists both or neither +# (contracts/cultnet/interop/write-receipt.md, "Capability: opting in"). +CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS: tuple[str, ...] = ("cultnet.write_admitted.v0", "cultnet.error.v1") + _ERROR_DETAILS_KEYS = ("field", "value", "asOf", "current") @@ -54,16 +65,48 @@ def _is_uint(value: Any) -> bool: return isinstance(value, int) and not isinstance(value, bool) and value >= 0 -def _validate_error_body(body: dict[str, Any]) -> None: - """Refuse a cultnet.error.v0 body that is not the full-key shape the schema requires.""" - expected = ("error", "routingHint", "code", "details") +def _is_name(value: Any) -> bool: + return isinstance(value, str) and value != "" + + +def _validate_write_admitted_body(body: dict[str, Any]) -> None: + """Refuse a cultnet.write_admitted.v0 body that is not exactly the five keys of its closed schema.""" + expected = ("messageId", "operation", "schemaId", "recordKey") if set(body) != set(expected): - raise ValueError(f"cultnet.error.v0 must carry exactly {list(expected)}, got {sorted(body)}") + raise ValueError(f"cultnet.write_admitted.v0 must carry exactly {list(expected)}, got {sorted(body)}") + for key in ("messageId", "schemaId", "recordKey"): + if not _is_name(body[key]): + raise ValueError(f"cultnet.write_admitted.v0 {key} must be a non-empty string") + if body["operation"] not in ("put", "delete"): + raise ValueError("cultnet.write_admitted.v0 operation must be put or delete") + + +def write_content_hash(operation: str, schema_id: str, record_key: str, payload: bytes | None) -> bytes: + """SHA-256 of the MessagePack array [operation, schemaId, recordKey, payload] (write-receipt.md, rule 5). + + payload is the raw document payload, packed as a MessagePack bin for a put and nil for a delete. + """ + return hashlib.sha256(msgpack.packb([operation, schema_id, record_key, payload], use_bin_type=True)).digest() + + +def hello_advertises_write_receipts(hello_message: CultNetMessage) -> bool: + """True when a hello lists both receipt frames: the only thing that makes a connection receipt-capable.""" + listed = hello_message.body.get("supportedMessageVersions") + return isinstance(listed, list) and all(version in listed for version in CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS) + + +def _validate_error_body(body: dict[str, Any], version: str, codes: tuple[str, ...]) -> None: + """Refuse a cultnet.error body that is not the full-key shape its closed schema requires.""" + expected = ("error", "routingHint", "code", "details") + (("messageId",) if version == "cultnet.error.v1" else ()) + if set(body) != set(expected): + raise ValueError(f"{version} must carry exactly {list(expected)}, got {sorted(body)}") if not isinstance(body["error"], str) or not body["error"]: - raise ValueError("cultnet.error.v0 error must be a non-empty string") + raise ValueError(f"{version} error must be a non-empty string") code = body["code"] - if code is not None and (not isinstance(code, str) or code not in CULTNET_ERROR_CODES): - raise ValueError(f"cultnet.error.v0 code {code!r} is not a recognized refusal code") + if code is not None and (not isinstance(code, str) or code not in codes): + raise ValueError(f"{version} code {code!r} is not a recognized refusal code") + if "messageId" in body and body["messageId"] is not None and not _is_name(body["messageId"]): + raise ValueError(f"{version} messageId must be a non-empty string or null") hint = body["routingHint"] if hint is not None: if ( @@ -72,17 +115,17 @@ def _validate_error_body(body: dict[str, Any]) -> None: or not isinstance(hint["reason"], str) or not (hint["shard"] is None or isinstance(hint["shard"], dict)) ): - raise ValueError("cultnet.error.v0 routingHint must be null or {shard, reason}") + raise ValueError(f"{version} routingHint must be null or {{shard, reason}}") details = body["details"] if details is not None: if not isinstance(details, dict) or set(details) != set(_ERROR_DETAILS_KEYS): - raise ValueError(f"cultnet.error.v0 details must be null or carry exactly {list(_ERROR_DETAILS_KEYS)}") + raise ValueError(f"{version} details must be null or carry exactly {list(_ERROR_DETAILS_KEYS)}") for key in ("field", "value"): if details[key] is not None and not isinstance(details[key], str): - raise ValueError(f"cultnet.error.v0 details.{key} must be a string or null") + raise ValueError(f"{version} details.{key} must be a string or null") for key in ("asOf", "current"): if details[key] is not None and not _is_uint(details[key]): - raise ValueError(f"cultnet.error.v0 details.{key} must be a non-negative integer or null") + raise ValueError(f"{version} details.{key} must be a non-negative integer or null") def hello( @@ -92,17 +135,18 @@ def hello( supported_schema_versions: list[str] | None = None, supported_mutation_contracts: list[dict[str, Any]] | None = None, transport_profiles: list[dict[str, Any]] | None = None, + supported_message_versions: list[str] | None = None, ) -> CultNetMessage: - return CultNetMessage( - "cultnet.hello.v0", - { - "runtimeId": runtime_id, - "supportedWireContracts": supported_wire_contracts or ["cultnet.schema.v0"], - "supportedSchemaVersions": supported_schema_versions or [], - "supportedMutationContracts": supported_mutation_contracts or [], - "transportProfiles": transport_profiles or [], - }, - ) + body: dict[str, Any] = { + "runtimeId": runtime_id, + "supportedWireContracts": supported_wire_contracts or ["cultnet.schema.v0"], + "supportedSchemaVersions": supported_schema_versions or [], + "supportedMutationContracts": supported_mutation_contracts or [], + "transportProfiles": transport_profiles or [], + } + if supported_message_versions is not None: + body["supportedMessageVersions"] = supported_message_versions + return CultNetMessage("cultnet.hello.v0", body) def login( @@ -172,7 +216,7 @@ def error_message( } if details is not None and set(details) - set(_ERROR_DETAILS_KEYS): raise ValueError(f"cultnet.error.v0 details has unknown keys {sorted(set(details) - set(_ERROR_DETAILS_KEYS))}") - _validate_error_body(body) + _validate_error_body(body, "cultnet.error.v0", CULTNET_ERROR_CODES) return CultNetMessage("cultnet.error.v0", body) diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py new file mode 100644 index 00000000..32c5f03a --- /dev/null +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import json +import unittest +from pathlib import Path +from typing import Any + +import msgpack + +from cultnet_py import ( + CULTNET_ERROR_CODES, + CULTNET_ERROR_V1_CODES, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + error_message, + hello, + hello_advertises_write_receipts, + parse_message, + write_content_hash, +) + +# Decoder-side conformance with contracts/cultnet/interop/write-receipt-vectors.json: `contentHash`, `wire` and +# `schemaInvalid`. `dedupe` and `blankFields` are server obligations and `scenarios` is the client write tracker; +# this runtime has neither a receipt server nor a write client, so it replays none of them. +_INTEROP = Path(__file__).resolve().parents[3] / "contracts/cultnet/interop" +_VECTORS = json.loads((_INTEROP / "write-receipt-vectors.json").read_text(encoding="utf-8")) +_ERROR_V1_SCHEMA = json.loads((_INTEROP / "cultnet.error.v1.schema.json").read_text(encoding="utf-8")) + + +def _pack(wire: dict[str, Any]) -> bytes: + return msgpack.packb(wire, use_bin_type=True) + + +def _admitted(**override: Any) -> dict[str, Any]: + return { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "put-1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1", + **override, + } + + +def _refused(**override: Any) -> dict[str, Any]: + return { + "schemaVersion": "cultnet.error.v1", + "error": "write_not_authorized: no", + "routingHint": None, + "code": "write_not_authorized", + "details": None, + "messageId": "put-1", + **override, + } + + +class WriteReceiptVectorTests(unittest.TestCase): + def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: + self.assertEqual( + set(_VECTORS) - {"provenance", "specification"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios"}, + ) + self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) + self.assertEqual(len(_VECTORS["wire"]), 7) + self.assertEqual(len(_VECTORS["schemaInvalid"]), 7) + + def test_content_hash_matches_every_vector(self) -> None: + for case in _VECTORS["contentHash"]["cases"]: + payload = None if case["payloadHex"] is None else bytes.fromhex(case["payloadHex"]) + digest = write_content_hash(case["operation"], case["schemaId"], case["recordKey"], payload) + self.assertEqual(digest.hex(), case["sha256"], case["name"]) + + def test_wire_vectors_decode_to_their_fields_in_order_and_reencode_byte_identical(self) -> None: + for vector in _VECTORS["wire"]: + raw = bytes.fromhex(vector["messagePackHex"]) + message = parse_message(raw) + self.assertEqual(message.to_wire(), vector["message"], vector["name"]) + self.assertEqual(list(message.to_wire()), list(vector["message"]), vector["name"]) + self.assertEqual(message.to_bytes(), raw, vector["name"]) + + def test_schema_invalid_vectors_are_refused(self) -> None: + for vector in _VECTORS["schemaInvalid"]: + with self.assertRaises(ValueError, msg=vector["name"]): + parse_message(bytes.fromhex(vector["messagePackHex"])) + + def test_a_payload_that_is_not_bytes_changes_the_hash(self) -> None: + # The payload is a bin, not an str and not a re-parsed document. + as_bin = write_content_hash("put", "note.v0", "note:1", b"\x81\xa2id\xa11") + self.assertNotEqual(as_bin, write_content_hash("put", "note.v0", "note:1", None)) + self.assertEqual(len(as_bin), 32) + + +class ErrorV1Tests(unittest.TestCase): + def test_code_set_equals_the_schema_enum_and_v0_is_unchanged(self) -> None: + schema_codes = [c for c in _ERROR_V1_SCHEMA["properties"]["code"]["enum"] if c is not None] + self.assertEqual(sorted(CULTNET_ERROR_V1_CODES), sorted(schema_codes)) + self.assertNotIn("limit_reached", CULTNET_ERROR_CODES) + self.assertEqual(CULTNET_ERROR_V1_CODES[-1], "limit_reached") + + def test_every_v1_code_decodes_and_an_unknown_one_does_not(self) -> None: + for code in CULTNET_ERROR_V1_CODES: + self.assertEqual(parse_message(_pack(_refused(code=code))).body["code"], code) + parse_message(_pack(_refused(code=None))) + with self.assertRaises(ValueError): + parse_message(_pack(_refused(code="invented"))) + with self.assertRaises(ValueError): + parse_message(_pack(_refused(code=7))) + + def test_limit_reached_is_v1_only(self) -> None: + v0 = {k: v for k, v in _refused(code="limit_reached", schemaVersion="cultnet.error.v0").items() if k != "messageId"} + with self.assertRaises(ValueError): + parse_message(_pack(v0)) + with self.assertRaises(ValueError): + error_message("x", code="limit_reached") + + def test_message_id_is_required_and_is_a_nonempty_string_or_null(self) -> None: + self.assertEqual(parse_message(_pack(_refused(messageId="w-é 1"))).body["messageId"], "w-é 1") + self.assertIsNone(parse_message(_pack(_refused(messageId=None))).body["messageId"]) + omitted = {k: v for k, v in _refused().items() if k != "messageId"} + with self.assertRaises(ValueError): + parse_message(_pack(omitted)) + for bad in ("", 5, b"put-1", ["put-1"], True): + with self.assertRaises(ValueError, msg=repr(bad)): + parse_message(_pack(_refused(messageId=bad))) + + def test_v0_stays_closed_to_message_id(self) -> None: + v0 = _refused(schemaVersion="cultnet.error.v0", code="message_invalid") + with self.assertRaises(ValueError): + parse_message(_pack(v0)) + del v0["messageId"] + self.assertEqual(parse_message(_pack(v0)).schema_version, "cultnet.error.v0") + + def test_v1_is_closed_to_unknown_keys_and_keeps_the_shape_rules_of_v0(self) -> None: + for bad in ( + _refused(extra=1), + _refused(error=""), + _refused(error=3), + _refused(routingHint={"shard": None}), + _refused(routingHint={"shard": 1, "reason": "r"}), + _refused(routingHint={"shard": None, "reason": 1}), + _refused(details={"field": None, "value": None, "asOf": None}), + _refused(details={"field": 1, "value": None, "asOf": None, "current": None}), + _refused(details={"field": None, "value": None, "asOf": -1, "current": None}), + _refused(details={"field": None, "value": None, "asOf": None, "current": True}), + ): + with self.assertRaises(ValueError, msg=str(bad)): + parse_message(_pack(bad)) + for good in ( + _refused(routingHint={"shard": None, "reason": "r"}), + _refused(details={"field": "messageId", "value": "x", "asOf": 1, "current": 2}), + ): + parse_message(_pack(good)) + + def test_the_refusal_reason_names_its_version(self) -> None: + with self.assertRaisesRegex(ValueError, "cultnet.error.v1 code"): + parse_message(_pack(_refused(code="invented"))) + with self.assertRaisesRegex(ValueError, r"routingHint must be null or \{shard, reason\}"): + parse_message(_pack(_refused(routingHint={"shard": None}))) + + +class RefusalTextTests(unittest.TestCase): + """A refusal names its frame version and the rule it broke; a caller reads this text in a log.""" + + def assertRefused(self, wire: dict[str, Any], pattern: str) -> None: + with self.assertRaisesRegex(ValueError, pattern): + parse_message(_pack(wire)) + + def test_error_frames_name_their_own_version(self) -> None: + v0 = {k: v for k, v in _refused(schemaVersion="cultnet.error.v0").items() if k != "messageId"} + self.assertRefused({**v0, "code": "nope"}, r"^cultnet\.error\.v0 code") + self.assertRefused({**v0, "extra": 1}, r"^cultnet\.error\.v0 must carry exactly") + self.assertRefused(_refused(extra=1), r"^cultnet\.error\.v1 must carry exactly") + self.assertRefused(_refused(error=""), r"^cultnet\.error\.v1 error must be") + self.assertRefused(_refused(messageId=""), r"^cultnet\.error\.v1 messageId must be") + self.assertRefused(_refused(details={"field": None}), r"^cultnet\.error\.v1 details must be null or carry exactly") + self.assertRefused( + _refused(details={"field": 1, "value": None, "asOf": None, "current": None}), r"^cultnet\.error\.v1 details\.field must be" + ) + self.assertRefused( + _refused(details={"field": None, "value": None, "asOf": -1, "current": None}), r"^cultnet\.error\.v1 details\.asOf must be" + ) + with self.assertRaisesRegex(ValueError, r"^cultnet\.error\.v0 code"): + error_message("x", code="bogus") + + def test_write_admitted_names_the_broken_rule(self) -> None: + self.assertRefused(_admitted(extra=1), r"^cultnet\.write_admitted\.v0 must carry exactly") + self.assertRefused(_admitted(recordKey=""), r"^cultnet\.write_admitted\.v0 recordKey must be a non-empty string") + self.assertRefused(_admitted(operation="upsert"), r"^cultnet\.write_admitted\.v0 operation must be put or delete$") + + +class WriteAdmittedTests(unittest.TestCase): + def test_both_operations_decode(self) -> None: + for operation in ("put", "delete"): + self.assertEqual(parse_message(_pack(_admitted(operation=operation))).body["operation"], operation) + + def test_every_key_is_required_and_no_other_is_allowed(self) -> None: + for key in ("messageId", "operation", "schemaId", "recordKey"): + with self.assertRaises(ValueError, msg=key): + parse_message(_pack({k: v for k, v in _admitted().items() if k != key})) + for extra in ("error", "code", "details"): + with self.assertRaises(ValueError, msg=extra): + parse_message(_pack(_admitted(**{extra: "x"}))) + + def test_each_string_is_nonempty_and_a_string(self) -> None: + for key in ("messageId", "schemaId", "recordKey"): + for bad in ("", None, 3, b"x"): + with self.assertRaises(ValueError, msg=f"{key}={bad!r}"): + parse_message(_pack(_admitted(**{key: bad}))) + + def test_operation_is_put_or_delete(self) -> None: + for bad in ("upsert", "PUT", "", None, 1): + with self.assertRaises(ValueError, msg=repr(bad)): + parse_message(_pack(_admitted(operation=bad))) + + def test_message_id_is_echoed_byte_for_byte(self) -> None: + self.assertEqual(parse_message(_pack(_admitted(messageId=" w é 1 "))).body["messageId"], " w é 1 ") + + +class HelloCapabilityTests(unittest.TestCase): + def test_capability_needs_both_frames(self) -> None: + both = list(CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS) + self.assertEqual(both, ["cultnet.write_admitted.v0", "cultnet.error.v1"]) + self.assertTrue(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=both))) + self.assertTrue(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=[*both, "x"]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=both[:1]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=both[1:]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=[]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r"))) + + def test_a_non_list_listing_is_not_a_capability(self) -> None: + wire = hello(runtime_id="r").to_wire() + wire["supportedMessageVersions"] = "cultnet.write_admitted.v0 cultnet.error.v1" + self.assertFalse(hello_advertises_write_receipts(parse_message(_pack(wire)))) + + def test_hello_writes_the_listing_only_when_given_and_round_trips(self) -> None: + self.assertNotIn("supportedMessageVersions", hello(runtime_id="r").body) + listed = hello(runtime_id="r", supported_message_versions=list(CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS)) + self.assertEqual(parse_message(listed.to_bytes()), listed) + + +if __name__ == "__main__": + unittest.main() From 56b0c1679c4bc07940dcbcd69d773edb6fa27ecc Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:55 +0000 Subject: [PATCH 075/117] cultmesh-rs: receipt_interop_server example, the real RUDP document server on an ephemeral port for the interop harness Co-Authored-By: Claude Sonnet 5.5 --- .../examples/receipt_interop_server.rs | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 packages/cultmesh-rs/examples/receipt_interop_server.rs diff --git a/packages/cultmesh-rs/examples/receipt_interop_server.rs b/packages/cultmesh-rs/examples/receipt_interop_server.rs new file mode 100644 index 00000000..b8c66fd9 --- /dev/null +++ b/packages/cultmesh-rs/examples/receipt_interop_server.rs @@ -0,0 +1,77 @@ +//! The real Rust RUDP document server on an ephemeral loopback port, for the cross-runtime write-receipt interop +//! harness (packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts). It is the production server with a +//! recording sink and nothing else: stdout carries one JSON line per fact the harness asserts on, and the server runs +//! until stdin closes. +//! +//! {"event":"listening","port":N} +//! {"event":"applied","op":"put","messageId":"...","schemaId":"...","recordKey":"..."} +//! +//! Record keys that start with `refuse:` are refused by the sink (authorize and commit), which is how the harness +//! asks for a refused write. +use anyhow::Result; +use cultmesh_rs::{ + CultMeshRudpDocumentServer, CultMeshRudpRawDocumentReceipt, CultMeshRudpRawDocumentSink, + CultMeshRudpSnapshotQuery, CultMeshSystemClock, +}; +use cultnet_rs::CultNetRawDocumentRecord; +use serde_json::json; +use std::io::Read; +use std::net::UdpSocket; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; +use std::time::Duration; + +const REFUSED_PREFIX: &str = "refuse:"; + +struct RecordingSink; + +impl CultMeshRudpRawDocumentSink for RecordingSink { + fn authorize_raw_document(&mut self, receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + if receipt.document.record_key.starts_with(REFUSED_PREFIX) { + anyhow::bail!("not authorized"); + } + Ok(()) + } + + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + if receipt.document.record_key.starts_with(REFUSED_PREFIX) { + anyhow::bail!("not authorized"); + } + println!( + "{}", + json!({ + "event": "applied", + "op": "put", + "messageId": receipt.message_id, + "schemaId": receipt.document.schema_id, + "recordKey": receipt.document.record_key, + }) + ); + Ok(()) + } +} + +fn main() -> Result<()> { + let source = |_: &CultMeshRudpSnapshotQuery| -> Result> { Ok(Vec::new()) }; + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + RecordingSink, + source, + CultMeshSystemClock::default(), + Default::default(), + )?; + println!("{}", json!({ "event": "listening", "port": server.local_addr()?.port() })); + let stop = Arc::new(AtomicBool::new(false)); + let watcher = Arc::clone(&stop); + thread::spawn(move || { + let mut sink = Vec::new(); + let _ = std::io::stdin().read_to_end(&mut sink); + watcher.store(true, Ordering::SeqCst); + }); + while !stop.load(Ordering::SeqCst) { + server.poll_once()?; + thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} From e4d2446dbe4dd268ee3b7bd20f686d72e66a53e3 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:55 +0000 Subject: [PATCH 076/117] C# InteropPeer: receipt-serve (CultNetDatabaseServer on an ephemeral port) and receipt-relay (LiteNetLib byte shim for the TS client) Co-Authored-By: Claude Sonnet 5.5 --- .../Program.cs | 8 +- .../ReceiptHarness.cs | 118 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs diff --git a/tests/GameCult.Networking.InteropPeer/Program.cs b/tests/GameCult.Networking.InteropPeer/Program.cs index 17a1ea89..fcfcf09c 100644 --- a/tests/GameCult.Networking.InteropPeer/Program.cs +++ b/tests/GameCult.Networking.InteropPeer/Program.cs @@ -25,7 +25,7 @@ static async Task ProgramMainAsync(string[] args) { if (args.Length == 0) { - throw new InvalidOperationException("Expected mode: serve | probe | dial | rudp-serve-once | rudp-dial-once | rudp-serve-message-once | rudp-dial-message-once | quic-realtime-serve"); + throw new InvalidOperationException("Expected mode: serve | probe | dial | rudp-serve-once | rudp-dial-once | rudp-serve-message-once | rudp-dial-message-once | quic-realtime-serve | receipt-serve | receipt-relay"); } var mode = args[0]; @@ -53,6 +53,12 @@ static async Task ProgramMainAsync(string[] args) case "rudp-dial-message-once": RudpDialMessageOnce(options); return 0; + case "receipt-serve": + await GameCult.Networking.InteropPeer.ReceiptHarness.ServeAsync(); + return 0; + case "receipt-relay": + await GameCult.Networking.InteropPeer.ReceiptHarness.RelayAsync(int.Parse(options["port"], CultureInfo.InvariantCulture)); + return 0; case "quic-realtime-serve": await QuicRealtimeServeAsync(ParseQuicRealtimeServeConfig(options)); return 0; diff --git a/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs new file mode 100644 index 00000000..75eaf4da --- /dev/null +++ b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs @@ -0,0 +1,118 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using LiteNetLib; +using R3; + +// The real C# receipt server (CultNetDatabaseServer on Server) and the one transport shim the TypeScript client needs to reach it, +// for the cross-runtime write-receipt interop harness (packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts). +// TypeScript has no LiteNetLib client, so the shim moves bytes and nothing else: each CultNet frame is a LiteNetLib reliable +// ordered payload, and on the pipe it is a 4-byte big-endian length followed by the frame. +namespace GameCult.Networking.InteropPeer +{ + [CultDocument("receipt.interop_note", ReceiptHarness.SchemaId)] + [MessagePack.MessagePackObject] + public sealed class ReceiptInteropNote + { + [MessagePack.Key(0)] + public string Text { get; set; } = string.Empty; + } + + internal static class ReceiptHarness + { + public const string SchemaId = "receipt.interop_note.v1"; + public const string ShardId = "receipt-interop-shard"; + // A record key with this prefix is refused by the authority: how the harness asks for a refused write. + public const string RefusedPrefix = "refuse:"; + + private sealed class Authority : ICultNetRemoteWriteAuthority + { + public CultNetRemoteWriteDecision Decide(CultNetRemoteWriteRequest request) => + request.RecordKey.ToString().StartsWith(RefusedPrefix, StringComparison.Ordinal) + ? CultNetRemoteWriteDecision.Refuse("denied") + : CultNetRemoteWriteDecision.Grant(); + } + + private static void Emit(object fact) + { + lock (Console.Out) Console.Out.WriteLine(JsonSerializer.Serialize(fact)); + } + + // stdout: {"event":"listening","port":N} then {"event":"applied"|"removed","recordKey":"..."} per committed change; runs until stdin closes. + public static async Task ServeAsync() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId])] + }); + using var server = new Server(cache, ServerSecurityOptions.Development()) { Port = 0 }; + using var databaseServer = new CultNetDatabaseServer(server, database, new CultNetDatabaseServerOptions + { + RemoteWriteAuthority = new Authority() + }); + using var watch = cache.Watch().Subscribe(change => + Emit(new { @event = change.Kind == CultCacheDocumentChangeKind.Removed ? "removed" : "applied", recordKey = change.Key.ToString() })); + server.Start(); + Emit(new { @event = "listening", port = server.Port, shardId = ShardId, schemaId = SchemaId }); + await Task.Run(() => Console.In.ReadToEnd()); + } + + // Connects to the server on `port` and relays frames between stdin/stdout and the connection; exits when either ends. + public static async Task RelayAsync(int port) + { + var listener = new EventBasedNetListener(); + var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ended = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var output = Console.OpenStandardOutput(); + var gate = new object(); + listener.PeerConnectedEvent += _ => connected.TrySetResult(true); + listener.PeerDisconnectedEvent += (_, _) => ended.TrySetResult(true); + listener.NetworkReceiveEvent += (_, reader, _, _) => + { + var frame = reader.GetRemainingBytes(); + var header = new byte[] { (byte)(frame.Length >> 24), (byte)(frame.Length >> 16), (byte)(frame.Length >> 8), (byte)frame.Length }; + lock (gate) + { + output.Write(header); + output.Write(frame); + output.Flush(); + } + }; + var manager = new NetManager(listener) { UnsyncedEvents = true }; + manager.Start(0); + var peer = manager.Connect("127.0.0.1", port, ServerSecurityOptions.Development().ConnectionKey); + if (await Task.WhenAny(connected.Task, Task.Delay(TimeSpan.FromSeconds(10))) != connected.Task) + throw new InvalidOperationException("the relay did not connect"); + var input = Console.OpenStandardInput(); + var pump = Task.Run(() => + { + var header = new byte[4]; + while (ReadExactly(input, header)) + { + var frame = new byte[(header[0] << 24) | (header[1] << 16) | (header[2] << 8) | header[3]]; + if (!ReadExactly(input, frame)) break; + peer.Send(frame, DeliveryMethod.ReliableOrdered); + } + }); + await Task.WhenAny(pump, ended.Task); + manager.Stop(); + } + + private static bool ReadExactly(Stream stream, byte[] buffer) + { + var read = 0; + while (read < buffer.Length) + { + var count = stream.Read(buffer, read, buffer.Length - read); + if (count <= 0) return false; + read += count; + } + return true; + } + } +} From 140eca51384ddac73c61dff68afa2bee5913881c Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:55 +0000 Subject: [PATCH 077/117] cultmesh-ts: TS write-receipt client against the real Rust and C# receipt servers (test:interop) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 + packages/cultmesh-ts/package.json | 3 +- .../interop/write-receipt-interop.test.ts | 418 ++++++++++++++++++ 3 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 35756ecf..4b13aa72 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -78,6 +78,8 @@ Those clients complete *successfully* when the window ends with no refusal. Unde Advertising is per server class, not per transport. A class advertises both frames in its hello only if it implements the receipt rules end to end; the C# RUDP schema server stays "no" until it answers writes with receipts. +Cross-runtime check: `packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts` (`npm run test:interop` in `packages/cultmesh-ts`) runs the TypeScript client (`WriteTracker`, the decoder, the one-shot writer) against the real Rust RUDP document server and the real `CultNetDatabaseServer` over loopback, one scenario set against both. Known gap it records as a todo: the Rust RUDP document server refuses every delete (`write_not_authorized: the server does not take deletes`), where this spec has an admitted delete receipted like a put. + ## Conformance follow-ups (not in this PR) C# server and client, Rust, TypeScript, Python, Kotlin: emit and parse the two frames; hello capability; tri-state client outcome; replace the #55 to #58 window-success; re-capture the wire vectors in each runtime. Server write paths include the shard write forwarder (rule 6) and the Odin catalog. diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index 92962cbc..0fef7419 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -24,7 +24,8 @@ "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", - "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --expose-gc --test dist-test/test/*.test.js" + "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --expose-gc --test dist-test/test/*.test.js", + "test:interop": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/write-receipt-interop.test.js" }, "keywords": [ "cultmesh", diff --git a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts new file mode 100644 index 00000000..8a646a50 --- /dev/null +++ b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts @@ -0,0 +1,418 @@ +// The TypeScript write-receipt client (cultnet-ts WriteTracker and decoder, cultmesh-ts one-shot writer) against the two real +// receipt servers over loopback, as child processes on ephemeral ports: +// +// rust packages/cultmesh-rs/examples/receipt_interop_server.rs, the RUDP document server, reached over RUDP. +// csharp tests/GameCult.Networking.InteropPeer `receipt-serve`, CultNetDatabaseServer on Server (LiteNetLib). TypeScript has no +// LiteNetLib client, so a second child, `receipt-relay`, carries bytes between the test and the server (4-byte +// big-endian length, then the CultNet frame: CultNetPeer's own stream framing). It decodes and judges nothing. +// +// Every scenario runs against both targets; its outcome is read from the WriteTracker (admitted, refused, unconfirmed), and the +// server's own stdout facts say what was committed. Contract: contracts/cultnet/interop/write-receipt.md and its vectors. +// +// Needs cargo and dotnet; run it with `npm run test:interop` in packages/cultmesh-ts (CARGO and DOTNET override the commands; +// RECEIPT_INTEROP_PREBUILT=1 skips the two builds). +import assert from "node:assert/strict"; +import { execFile, spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { Duplex } from "node:stream"; +import { after, before, describe, test } from "node:test"; +import { promisify } from "node:util"; +import { encode } from "@msgpack/msgpack"; +import { z } from "zod"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + CultNetPeer, + WriteTracker, + defineCultNetDocumentBinding, + helloAdvertisesWriteReceipts, + resultWindowMs, + type CultNetErrorV1Message, + type CultNetHelloMessage, + type CultNetMessage, + type WriteSettlement, +} from "@gamecult/cultnet-ts"; +import { CultMesh, CultMeshRudpWriteRefusedError, CultMeshRudpWriteUnconfirmedError } from "../../src/index"; + +const execFileAsync = promisify(execFile); +// dist-test/test/interop -> the repository root +const repoRoot = resolve(__dirname, "..", "..", "..", "..", ".."); +const cargo = process.env.CARGO ?? "cargo"; +const dotnet = process.env.DOTNET ?? "dotnet"; +const rustManifestDir = join(repoRoot, "packages", "cultmesh-rs"); +const csharpProject = join(repoRoot, "tests", "GameCult.Networking.InteropPeer", "GameCult.Networking.InteropPeer.csproj"); +const csharpDll = join(repoRoot, "bin", "GameCult.Networking.InteropPeer", "Debug", "net10.0", "GameCult.Networking.InteropPeer.dll"); +// Set when the server binaries were built beforehand (the builds themselves cannot start under a tight address-space cap). +const prebuilt = process.env.RECEIPT_INTEROP_PREBUILT === "1"; +const vectors = JSON.parse(readFileSync(join(repoRoot, "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); + +const SCHEMA_ID = "receipt.interop_note.v1"; +const UNREADABLE = "message_invalid: the message could not be read"; +const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms)); + +async function waitFor(predicate: () => boolean, what: string, ms = 5_000): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > ms) throw new Error(`timed out waiting for ${what}`); + await sleep(5); + } +} + +// ---- the servers ------------------------------------------------------------------------------------------------------- + +interface Fact { event: string; recordKey?: string; messageId?: string; [key: string]: unknown } + +/** A running server child and the JSON facts it prints on stdout. */ +class ServerProcess { + readonly facts: Fact[] = []; + stderr = ""; + port = 0; + constructor(readonly child: ChildProcess) { + let pending = ""; + child.stdout!.on("data", (chunk: Buffer) => { + pending += chunk.toString("utf8"); + for (let at = pending.indexOf("\n"); at >= 0; at = pending.indexOf("\n")) { + const line = pending.slice(0, at).trim(); + pending = pending.slice(at + 1); + if (line) this.facts.push(JSON.parse(line) as Fact); + } + }); + child.stderr!.on("data", (chunk: Buffer) => { this.stderr += chunk.toString("utf8"); }); + } + /** How many times `event` was printed for `recordKey`. */ + count(event: string, recordKey: string): number { + return this.facts.filter((fact) => fact.event === event && fact.recordKey === recordKey).length; + } + async stop(): Promise { + this.child.stdin!.end(); + await new Promise((done) => { + const killer = setTimeout(() => this.child.kill("SIGKILL"), 5_000); + this.child.once("exit", () => { clearTimeout(killer); done(); }); + if (this.child.exitCode !== null) done(); + }); + } +} + +async function launch(command: string, args: string[]): Promise { + const server = new ServerProcess(spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] })); + await waitFor(() => server.facts.some((fact) => fact.event === "listening") || server.child.exitCode !== null, "the server to listen", 60_000); + const listening = server.facts.find((fact) => fact.event === "listening"); + if (!listening) throw new Error(`the server exited before listening:\n${server.stderr}`); + server.port = listening.port as number; + return server; +} + +/** One connection to a server: the peer that speaks CultNet, and how to close it. */ +interface Connection { peer: CultNetPeer; close(): void } + +interface Target { + name: string; + /** Shard fields the server's database requires on a write, if any. */ + shard?: { shardId: string; shardEpoch: number }; + /** A document payload this target's store reads. */ + payload(text: string): Uint8Array; + start(): Promise; + connect(server: ServerProcess): Promise; +} + +let nextConnectionId = 0x20206000; + +const rust: Target = { + name: "rust", + payload: (text) => encode({ text }), + async start() { + if (!prebuilt) await execFileAsync(cargo, ["build", "--example", "receipt_interop_server", "--message-format", "short"], { cwd: rustManifestDir }); + const metadata = spawnSync(cargo, ["metadata", "--format-version", "1", "--no-deps"], { cwd: rustManifestDir, encoding: "utf8" }); + if (metadata.status !== 0) throw new Error(`cargo metadata failed: ${metadata.stderr}`); + const exe = join(JSON.parse(metadata.stdout).target_directory, "debug", "examples", `receipt_interop_server${process.platform === "win32" ? ".exe" : ""}`); + return launch(exe, []); + }, + async connect(server) { + const peer = await CultMesh.createRudpPeer("receipt-interop", nextConnectionId++, { host: "127.0.0.1", port: server.port, uri: `rudp://127.0.0.1:${server.port}` }); + return { peer, close: () => peer.close() }; + }, +}; + +const csharp: Target = { + name: "csharp", + shard: { shardId: "receipt-interop-shard", shardEpoch: 1 }, + payload: (text) => encode([text]), + async start() { + if (!prebuilt) await execFileAsync(dotnet, ["build", csharpProject, "-nologo", "-v", "q"], { cwd: repoRoot, maxBuffer: 64 * 1024 * 1024 }); + return launch(dotnet, [csharpDll, "receipt-serve"]); + }, + async connect(server) { + const relay = spawn(dotnet, [csharpDll, "receipt-relay", "--port", String(server.port)], { stdio: ["pipe", "pipe", "ignore"] }); + const peer = new CultNetPeer(Duplex.from({ readable: relay.stdout!, writable: relay.stdin! }), { wireContract: "cultnet.schema.v0" }); + return { peer, close: () => { relay.stdin!.end(); relay.kill(); } }; + }, +}; + +// ---- a client session: one connection, the frames it saw, one WriteTracker per write ----------------------------------------- + +class Session { + readonly frames: CultNetMessage[] = []; + readonly invalid: Error[] = []; + private readonly trackers = new Set(); + private serverReceipts = false; + private constructor(readonly target: Target, private readonly connection: Connection) { + connection.peer.on("message", (message: CultNetMessage) => { + if (message.schemaVersion === "cultnet.hello.v0") this.serverReceipts = helloAdvertisesWriteReceipts(message); + this.frames.push(message); + for (const tracker of this.trackers) tracker.frame(message, Date.now()); + }); + connection.peer.on("invalidMessage", (error: Error) => this.invalid.push(error)); + connection.peer.on("close", () => { for (const tracker of this.trackers) tracker.sessionEnd(Date.now()); }); + } + + static async open(target: Target, server: ServerProcess): Promise { + return new Session(target, await target.connect(server)); + } + + /** Declares the receipt capability and waits for the server's own hello. */ + async hello(versions: readonly string[] = CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS): Promise { + this.connection.peer.sendHello({ schemaVersion: "cultnet.hello.v0", runtimeId: "receipt-interop", runtimeKind: "cultmesh-ts", supportedMessageVersions: [...versions] }); + await waitFor(() => this.frames.some((frame) => frame.schemaVersion === "cultnet.hello.v0"), "the server's hello"); + return this.frames.find((frame) => frame.schemaVersion === "cultnet.hello.v0") as CultNetHelloMessage; + } + + put(messageId: string, recordKey: string, text: string, document: Record = {}): CultNetMessage { + return this.shard({ + schemaVersion: "cultnet.document_put_raw.v0", + messageId, + document: { + schemaId: SCHEMA_ID, recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", + payload: this.target.payload(text), ...document, + }, + }); + } + + delete(messageId: string, recordKey: string): CultNetMessage { + return this.shard({ schemaVersion: "cultnet.document_delete.v0", messageId, schemaId: SCHEMA_ID, recordKey }); + } + + private shard(message: unknown): CultNetMessage { + return { ...(message as object), ...this.target.shard } as CultNetMessage; + } + + /** + * Sends one write and settles it with its own tracker, windowed by what the server advertised (5000 ms against a receipt + * server, 500 ms otherwise). Only the tracker decides the outcome. + */ + async write(message: CultNetMessage): Promise { + const messageId = (message as { messageId: string }).messageId; + const tracker = new WriteTracker({ windowMs: resultWindowMs(this.serverReceipts) }); + this.trackers.add(tracker); + tracker.send(messageId, Date.now()); + this.connection.peer.send(message); + for (;;) { + tracker.tick(Date.now()); + const state = tracker.state(messageId)!; + if (state.outcome !== "pending") return state; + await sleep(Math.min(20, tracker.nextDeadlineMs(Date.now()) ?? 20)); + } + } + + /** Frames of `schemaVersion` that name `messageId`. */ + answers(messageId: string, schemaVersion: string): CultNetMessage[] { + return this.frames.filter((frame) => frame.schemaVersion === schemaVersion && (frame as { messageId?: string }).messageId === messageId); + } + + close(): void { this.connection.close(); } +} + +function refusal(settlement: WriteSettlement): CultNetErrorV1Message { + assert.equal(settlement.outcome, "refused", JSON.stringify(settlement)); + return (settlement as Extract).response as CultNetErrorV1Message; +} + +// ---- the scenarios, run against each server ----------------------------------------------------------------------------------- + +for (const target of [rust, csharp]) { + describe(`write receipt, TypeScript client against the real ${target.name} server`, () => { + let server: ServerProcess; + let session: Session | undefined; + before(async () => { server = await target.start(); }, { timeout: 180_000 }); + after(async () => { session?.close(); await server?.stop(); }); + + /** A fresh receipt session (hello sent, the server's hello read) for a scenario. */ + async function receiptSession(): Promise { + session?.close(); + session = await Session.open(target, server); + const hello = await session.hello(); + assert.ok(helloAdvertisesWriteReceipts(hello), "the server's hello lists both receipt frames"); + return session; + } + + test("hello listing both frames: the server answers with its own hello listing both, then a put is receipted after it", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + assert.equal(s.frames[0].schemaVersion, "cultnet.hello.v0"); + const settlement = await s.write(s.put("hello-1", "hello:a", "one")); + assert.deepEqual(settlement, { outcome: "admitted" }); + assert.deepEqual(s.frames.map((frame) => frame.schemaVersion), ["cultnet.hello.v0", "cultnet.write_admitted.v0"]); + }); + + test("a hello listing only one of the two frames does not opt in: no receipt, and the put is applied", { timeout: 30_000 }, async () => { + for (const listed of CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS) { + session?.close(); + session = await Session.open(target, server); + session.hello([listed]).catch(() => undefined); // a server may stay silent to a hello that does not opt in + await sleep(300); + const settlement = await session.write(session.put(`half-${listed}`, `half:${listed}`, "one")); + assert.deepEqual(settlement, { outcome: "unconfirmed", reason: "window_elapsed" }, `listing only ${listed}`); + assert.ok(!session.frames.some((frame) => frame.schemaVersion === "cultnet.write_admitted.v0" || frame.schemaVersion === "cultnet.error.v1")); + await waitFor(() => server.count("applied", `half:${listed}`) === 1, "the put to be applied"); + } + }); + + test("put admitted: the receipt echoes the write and the store has it once", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const settlement = await s.write(s.put("put-1", "put:a", "one")); + assert.deepEqual(settlement, { outcome: "admitted" }); + assert.deepEqual(s.answers("put-1", "cultnet.write_admitted.v0"), [ + { schemaVersion: "cultnet.write_admitted.v0", messageId: "put-1", operation: "put", schemaId: SCHEMA_ID, recordKey: "put:a" }, + ]); + await waitFor(() => server.count("applied", "put:a") === 1, "the put to be applied"); + }); + + // The Rust RUDP document server has no delete: its sink only accepts documents, and every delete on a receipt session is + // answered with a refusal (rudp_document_server.rs, deletes_on_a_receipt_session_are_refused_one_by_one). The spec + // (Server rules 1 to 3) has a delete admitted and receipted like a put. + test("delete admitted: the receipt says operation delete and the store drops the record", { + timeout: 30_000, + todo: target.name === "rust" ? "the Rust RUDP document server refuses every delete; there is no delete port in CultMeshRudpRawDocumentSink" : false, + }, async () => { + const s = await receiptSession(); + assert.deepEqual(await s.write(s.put("del-put", "del:a", "one")), { outcome: "admitted" }); + assert.deepEqual(await s.write(s.delete("del-1", "del:a")), { outcome: "admitted" }); + assert.deepEqual(s.answers("del-1", "cultnet.write_admitted.v0"), [ + { schemaVersion: "cultnet.write_admitted.v0", messageId: "del-1", operation: "delete", schemaId: SCHEMA_ID, recordKey: "del:a" }, + ]); + await waitFor(() => server.count("removed", "del:a") === 1, "the delete to be applied"); + }); + + test("refused write: an error.v1 carrying the write's id, and nothing is applied", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const response = refusal(await s.write(s.put("deny-1", "refuse:a", "one"))); + assert.equal(response.schemaVersion, "cultnet.error.v1"); + assert.equal(response.messageId, "deny-1"); + assert.equal(response.code, "write_not_authorized"); + // The session is kept: the next write on it is admitted. + assert.deepEqual(await s.write(s.put("deny-2", "deny:b", "one")), { outcome: "admitted" }); + assert.equal(server.count("applied", "refuse:a"), 0); + }); + + test("resend of the same id and content on the same session: re-receipted, applied once", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + assert.deepEqual(await s.write(s.put("resend-1", "resend:a", "one")), { outcome: "admitted" }); + assert.deepEqual(await s.write(s.put("resend-1", "resend:a", "one")), { outcome: "admitted" }); + const receipts = s.answers("resend-1", "cultnet.write_admitted.v0"); + assert.equal(receipts.length, 2); + assert.deepEqual(receipts[0], receipts[1]); + await sleep(200); // a second application would be printed by now + assert.equal(server.count("applied", "resend:a"), 1); + }); + + test("same id with different content: message_invalid naming the id, never a receipt, the first write untouched", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + assert.deepEqual(await s.write(s.put("clash-1", "clash:a", "one")), { outcome: "admitted" }); + const response = refusal(await s.write(s.put("clash-1", "clash:a", "other"))); + assert.equal(response.code, "message_invalid"); + assert.equal(response.messageId, "clash-1"); + assert.equal(response.details?.field, "messageId"); + assert.equal(s.answers("clash-1", "cultnet.write_admitted.v0").length, 1, "no receipt for the changed write"); + await sleep(200); + assert.equal(server.count("applied", "clash:a"), 1); + }); + + // Each case of the vectors' blankFields section: a document that names nothing is refused as an unreadable frame, with the + // write's id and the fixed text, and never applied; a document that is merely missing optional fields is admitted. + for (const blank of vectors.blankFields.cases as { name: string; set: Record; outcome: "admitted" | "refused" }[]) { + test(`blank-field vector ${blank.name}: ${blank.outcome}`, { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const recordKey = `blank:${blank.name}`; + const { payloadHex, ...fields } = blank.set; + const message = s.put(`blank-${blank.name}`, recordKey, "one", fields); + const sent = (message as unknown as { document: Record }).document; + for (const [key, value] of Object.entries(fields)) if (value === null) delete sent[key]; // nil means absent on the wire here + if (payloadHex === "") sent.payload = new Uint8Array(); + const settlement = await s.write(message); + if (blank.outcome === "admitted") { + assert.deepEqual(settlement, { outcome: "admitted" }); + await waitFor(() => server.count("applied", recordKey) === 1, "the put to be applied"); + return; + } + const response = refusal(settlement); + assert.equal(response.code, "message_invalid"); + assert.equal(response.error, UNREADABLE); + assert.equal(response.details?.field, "message"); + assert.equal(response.messageId, `blank-${blank.name}`); + await sleep(200); + assert.equal(server.count("applied", recordKey), 0); + assert.equal(server.count("applied", "blank:"), 0); + }); + } + + test("a delete that names nothing is refused as unreadable with its id", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const message = s.delete("blank-del", "blank:del"); + (message as unknown as { schemaId: string }).schemaId = ""; + const response = refusal(await s.write(message)); + assert.equal(response.code, "message_invalid"); + assert.equal(response.error, UNREADABLE); + assert.equal(response.details?.field, "message"); + assert.equal(response.messageId, "blank-del"); + }); + + test("legacy client (no hello): no receipt frames, the write is applied, and its refusal is the old error.v0", { timeout: 30_000 }, async () => { + session?.close(); + session = await Session.open(target, server); + assert.deepEqual(await session.write(session.put("legacy-1", "legacy:a", "one")), { outcome: "unconfirmed", reason: "window_elapsed" }); + await waitFor(() => server.count("applied", "legacy:a") === 1, "the put to be applied"); + const refused = (await session.write(session.put("legacy-2", "refuse:legacy", "one"))) as Extract; + assert.equal(refused.outcome, "refused"); + assert.equal(refused.response.schemaVersion, "cultnet.error.v0"); + assert.deepEqual(session.frames.map((frame) => frame.schemaVersion), ["cultnet.error.v0"]); + assert.equal(session.invalid.length, 0); + }); + }); +} + +// ---- the one-shot writer, which is the real cultmesh-ts client path, against the Rust RUDP server ---------------------------- +// (The C# server speaks LiteNetLib, not RUDP: no RUDP client path reaches it.) + +describe("write receipt, cultmesh-ts publishRudpDocumentOnce against the real rust server", () => { + const note = defineDocumentType({ type: "receipt.interop_note", schemaId: SCHEMA_ID, schema: z.object({ text: z.string() }), name: "text" }); + const binding = defineCultNetDocumentBinding({ definition: note }); + let server: ServerProcess; + before(async () => { server = await rust.start(); }, { timeout: 180_000 }); + after(async () => { await server?.stop(); }); + const endpoint = () => ({ host: "127.0.0.1", port: server.port, uri: `rudp://127.0.0.1:${server.port}` }); + + test("writeReceipts: the put resolves admitted on the receipt, after the server applied it", { timeout: 30_000 }, async () => { + const outcome = await CultMesh.publishRudpDocumentOnce("receipt-interop", nextConnectionId++, endpoint(), binding, "once:a", { text: "one" }, { writeReceipts: true, messageId: "once-1" }); + assert.equal(outcome, "admitted"); + await waitFor(() => server.count("applied", "once:a") === 1, "the put to be applied"); + }); + + test("writeReceipts: a refused put rejects with the server's error.v1 naming the put", { timeout: 30_000 }, async () => { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("receipt-interop", nextConnectionId++, endpoint(), binding, "refuse:once", { text: "one" }, { writeReceipts: true, messageId: "once-2" }), + (error: unknown) => { + assert.ok(error instanceof CultMeshRudpWriteRefusedError); + assert.equal(error.response.schemaVersion, "cultnet.error.v1"); + assert.equal((error.response as CultNetErrorV1Message).messageId, "once-2"); + return true; + }, + ); + }); + + test("without writeReceipts the server stays silent: the put is unconfirmed, not admitted, though the server applied it", { timeout: 30_000 }, async () => { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("receipt-interop", nextConnectionId++, endpoint(), binding, "once:legacy", { text: "one" }, { messageId: "once-3" }), + (error: unknown) => error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed", + ); + await waitFor(() => server.count("applied", "once:legacy") === 1, "the put to be applied"); + }); +}); From 52204c68c9c779bc6d14f7c276a82d9f4630c095 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:26:28 +0000 Subject: [PATCH 078/117] write-receipt vectors: U+001C to U+001F and padded names are not blank Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt-vectors.json | 6 ++++-- contracts/cultnet/interop/write-receipt.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 1d83ea89..50e8a20f 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -111,10 +111,10 @@ } ], "blankFields": { - "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or made only of Unicode White_Space code points (the list in whiteSpaceCodePoints, the PropList property, the same in every runtime; U+200B, U+FEFF and U+180E are not White_Space). Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.", + "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or made only of Unicode White_Space code points (the list in whiteSpaceCodePoints, the PropList property, the same in every runtime; U+200B, U+FEFF, U+180E and U+001C to U+001F are not White_Space, though some platforms' whitespace tests accept some of them). Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.", "validPutPayloadHex": "81A16101", "whiteSpaceCodePoints": ["0009","000A","000B","000C","000D","0020","0085","00A0","1680","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","200A","2028","2029","202F","205F","3000"], - "notWhiteSpaceCodePoints": ["200B","200C","200D","2060","FEFF","180E"], + "notWhiteSpaceCodePoints": ["200B","200C","200D","2060","FEFF","180E","001C","001D","001E","001F"], "codePointCases": "For every code point in whiteSpaceCodePoints, each of schemaId, recordKey, storedAt, sourceRuntimeId, sourceAgentId, sourceRole and one tags entry, set alone to a string of just that code point (and to that code point repeated three times), is refused; for every code point in notWhiteSpaceCodePoints the same string is not blank and the put is admitted.", "cases": [ { "name": "valid_document_is_admitted", "set": {}, "outcome": "admitted" }, @@ -142,6 +142,8 @@ { "name": "zero_width_space_tag_is_a_name", "set": { "tags": ["\u200b"] }, "outcome": "admitted" }, { "name": "bom_stored_at_is_a_name", "set": { "storedAt": "\ufeff" }, "outcome": "admitted" }, { "name": "mongolian_vowel_separator_source_role_is_a_name", "set": { "sourceRole": "\u180e" }, "outcome": "admitted" }, + { "name": "padded_name_is_a_name", "set": { "schemaId": "\u00a0a\u2003", "tags": [" a"] }, "outcome": "admitted" }, + { "name": "file_separator_schema_id_is_a_name", "set": { "schemaId": "\u001c" }, "outcome": "admitted" }, { "name": "source_ids_and_tags_present_and_named", "set": { "sourceRuntimeId": "r", "sourceAgentId": "a", "sourceRole": "w", "tags": ["a"] }, "outcome": "admitted" }, { "name": "source_ids_and_tags_nil", "set": { "sourceRuntimeId": null, "sourceAgentId": null, "sourceRole": null, "tags": null }, "outcome": "admitted" }, { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" } diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index a3ad68dc..4ef863e4 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -36,7 +36,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF and U+180E are not White_Space: a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. + - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (.NET's `char.IsWhiteSpace` accepts U+001C to U+001F, so a runtime on it must check the list explicitly): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. From 3b1b769762c9e7d966214fa095654eb68091d8b8 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:27:17 +0000 Subject: [PATCH 079/117] write-receipt spec: name the platforms that disagree accurately Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 4ef863e4..d3dc2614 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -36,7 +36,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests. - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table. - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (.NET's `char.IsWhiteSpace` accepts U+001C to U+001F, so a runtime on it must check the list explicitly): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. + - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. From 04207007d6b51d3be27b547300e1e2aac1718244 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 07:45:35 +0000 Subject: [PATCH 080/117] interop test: read the store under the record key the blank-field vector set A vector that sets recordKey (zero_width_space_record_key_is_a_name) replaces the harness key, so the admitted case waited for blank: and timed out against the Rust server, and the refused record-key cases counted applications under a key the server never saw. Use the key the document carried. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-ts/test/interop/write-receipt-interop.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts index 8a646a50..12bd4394 100644 --- a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts +++ b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts @@ -331,8 +331,9 @@ for (const target of [rust, csharp]) { for (const blank of vectors.blankFields.cases as { name: string; set: Record; outcome: "admitted" | "refused" }[]) { test(`blank-field vector ${blank.name}: ${blank.outcome}`, { timeout: 30_000 }, async () => { const s = await receiptSession(); - const recordKey = `blank:${blank.name}`; const { payloadHex, ...fields } = blank.set; + // A case that sets recordKey replaces the harness key, so the store is read under the key the server actually saw. + const recordKey = typeof fields.recordKey === "string" ? fields.recordKey : `blank:${blank.name}`; const message = s.put(`blank-${blank.name}`, recordKey, "one", fields); const sent = (message as unknown as { document: Record }).document; for (const [key, value] of Object.entries(fields)) if (value === null) delete sent[key]; // nil means absent on the wire here @@ -350,7 +351,6 @@ for (const target of [rust, csharp]) { assert.equal(response.messageId, `blank-${blank.name}`); await sleep(200); assert.equal(server.count("applied", recordKey), 0); - assert.equal(server.count("applied", "blank:"), 0); }); } From 4f6ee0db835aa1e6b7a5ce6d7e175ea463e75792 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:22:55 +0000 Subject: [PATCH 081/117] Rust RUDP server: pin the refusal of blank schemaId/recordKey on receipt sessions heimdall-low-rust-keys-untested. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index be667e72..e37ce9d6 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2983,6 +2983,75 @@ fn only_reliable_non_sequenced_writes_are_accepted_on_a_receipt_session() -> Res Ok(()) } +/// `bytes` with the MessagePack fixstr `from` replaced by the fixstr `to`, which may be blank. +fn swap_fixstr(bytes: &[u8], from: &str, to: &str) -> Vec { + assert!(from.len() < 32 && to.len() < 32); + let mut needle = vec![0xa0 | from.len() as u8]; + needle.extend_from_slice(from.as_bytes()); + let at = bytes.windows(needle.len()).position(|window| window == needle).expect("marker is in the frame"); + let mut out = bytes[..at].to_vec(); + out.push(0xa0 | to.len() as u8); + out.extend_from_slice(to.as_bytes()); + out.extend_from_slice(&bytes[at + needle.len()..]); + out +} + +/// A `schemaId` or `recordKey` that is present but empty or only whitespace names nothing: the write is refused +/// `message_invalid` like a frame that does not read (field "message", the write's id), never reaches the store, +/// and the session goes on (write-receipt.md, rule 5; vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`). +#[test] +fn a_write_whose_schema_id_or_record_key_names_nothing_is_refused_and_never_handed_to_the_store() -> Result<()> { + let blanks = ["", " ", "\t \n"]; + let mut connection_id = 300; + for blank in blanks { + for blank_field in ["schemaId", "recordKey"] { + for operation in ["put", "delete"] { + connection_id += 1; + let (schema_id, record_key) = match blank_field { + "schemaId" => (blank, "note:1"), + _ => ("test.raw.v1", blank), + }; + // The encoder refuses a blank field, so encode a marker and swap the blank into the bytes. + let bad = if operation == "put" { + let mut record = document("zz-key-zz", vec![1, 2, 3]); + record.schema_id = "zz-schema-zz".into(); + encode_cultnet_message_to_vec( + &CultNetMessage::DocumentPutRaw { message_id: "bad-1".into(), document: record }, + CultNetWireContract::CultNetSchemaV0, + )? + } else { + encode_cultnet_message_to_vec( + &CultNetMessage::DocumentDelete { + message_id: "bad-1".into(), + schema_id: "zz-schema-zz".into(), + record_key: "zz-key-zz".into(), + }, + CultNetWireContract::CultNetSchemaV0, + )? + }; + let bad = swap_fixstr(&bad, "zz-schema-zz", schema_id); + let bad = swap_fixstr(&bad, "zz-key-zz", record_key); + let label = format!("{operation} with {blank_field} {blank:?}"); + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, connection_id)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + peer.send_reliable("schema", bad)?; + send_reliable(&mut peer, &put("good-1", "note:2", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("bad-1".into())), "{label}"); + assert_eq!(error_text_and_field(&answers[0]).1, Some("message".into()), "{label}"); + assert_eq!(answers[1], admitted("good-1", "note:2"), "{label}: the session goes on"); + let state = sink.0.lock().unwrap(); + assert_eq!(state.authorized, vec!["good-1".to_string()], "{label}: the store was never asked about the blank write"); + assert_eq!(state.receipts.len(), 1, "{label}: only the good write was applied"); + } + } + } + Ok(()) +} + /// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured /// from the refusal and not from the packet before it. #[test] From bd2aff8ed0cdd76d6acc673dff5a0fea8fc10d72 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:23:07 +0000 Subject: [PATCH 082/117] Rust RUDP server: decide the sequenced-channel refusal per frame A sequenced packet's arrival can release held ordered frames in the same delivery; refuse only the packet's own frame. heimdall-low-refuse-sequenced-per-frame. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 4 +- .../cultmesh-rs/tests/rudp_document_server.rs | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index f97b2578..12fdf3e6 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -825,7 +825,9 @@ where if frame.channel_id != "schema" { continue; } - if refuse_sequenced { + // `delivered` also holds the ordered frames this packet's arrival released from the hold; those were + // not sent on the sequenced channel. They sit above the packet's own sequence, its own frame at or below. + if refuse_sequenced && frame.sequence <= packet.sequence { if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = self.refuse_unreliable_write(key, &frame.payload)? { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index e37ce9d6..b6663396 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3052,6 +3052,51 @@ fn a_write_whose_schema_id_or_record_key_names_nothing_is_refused_and_never_hand Ok(()) } +/// The sequenced-channel refusal belongs to the frame that arrived sequenced. A reliable-sequenced packet that +/// fills the gap an earlier ordered packet was held behind delivers both frames in one go: the sequenced write is +/// refused, the ordered write it released is admitted. +#[test] +fn a_sequenced_packet_releasing_a_held_ordered_write_refuses_only_its_own() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 250)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let mut packet_for = |peer: &mut RawPeer, id: &str, ordered: bool, sequenced: bool| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered, sequenced, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let sequenced = packet_for(&mut peer, "s1", false, true)?; + let ordered = packet_for(&mut peer, "o1", true, false)?; + // The ordered write arrives first, one sequence ahead of the gap, and is held. + for packet in &ordered { + peer.send_packet(packet)?; + } + server.poll_once()?; + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "the ordered write waits behind the gap"); + for packet in &sequenced { + peer.send_packet(packet)?; + } + server.poll_once()?; + let mut answers = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + let refusals: Vec<_> = answers.iter().filter(|a| matches!(a, CultNetMessage::ErrorV1 { .. })).collect(); + assert_eq!(refusals.len(), 1, "only the sequenced write is refused: {answers:?}"); + assert_eq!(refusal_of(refusals[0]), (Some(CultNetErrorCode::MessageInvalid), Some("s1".into()))); + assert!(answers.contains(&admitted("o1", "note:o1")), "the released ordered write is admitted: {answers:?}"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "and applied"); + Ok(()) +} + /// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured /// from the refusal and not from the packet before it. #[test] From 4e89f07bff194851d518c7cb921b27a8f3fe375a Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:13:24 +0000 Subject: [PATCH 083/117] Rust RUDP server: answer a superseded sequenced write; order frame and packet by serial arithmetic Superseded packet: a reliable-sequenced packet that arrives after a newer one delivers no frame, yet is acknowledged and never resent, so its write went unanswered. The test (red first: only "new" was refused, "old" got nothing) showed the drop happens in the session below frame decoding, but an unfragmented packet carries its whole frame, so the server can read the id from the packet payload. Chosen: refuse it from the packet payload. A superseded fragment of a larger write has no readable id in the packet, so write-receipt.md now says that write may go unanswered and the client's unconfirmed outcome covers it. Ordering: frame.sequence <= packet.sequence is replaced by sequence_at_or_before, serial arithmetic modulo 2^32. Deviation from the brief: cultnet-rs' at_or_before carries a 4096-sequence receive window made for duplicate detection; a frame's first fragment may lie further back than that, so reusing it would leave such a sequenced write unrefused and applied. The new test pins that, and a hand mutant with the window fails it. A wrapped sequence cannot reach this compare through the session today (it is judged a duplicate), so the unit test pins the helper, not an end-to-end wrap. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- .../cultmesh-rs/src/rudp_document_server.rs | 33 +++++- .../cultmesh-rs/tests/rudp_document_server.rs | 102 ++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index d3dc2614..b74411ea 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -38,7 +38,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. + - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 12fdf3e6..a0ab51ac 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -821,13 +821,26 @@ where } let mut kept_rejection = None; + // A superseded packet delivers no frame, yet it was acknowledged and will not be resent, so its write would + // go unanswered. A packet that carries its whole frame is answered from its own payload. A superseded + // fragment cannot be: the write's id may lie in a fragment this packet does not hold. + if refuse_sequenced + && packet.fragment_count == 0 + && !result.delivered.iter().any(|frame| frame.channel_id == "schema" && frame.sequence == packet.sequence) + { + if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = + self.refuse_unreliable_write(key, &packet.payload)? + { + kept_rejection = Some(rejection); + } + } for frame in result.delivered { if frame.channel_id != "schema" { continue; } // `delivered` also holds the ordered frames this packet's arrival released from the hold; those were // not sent on the sequenced channel. They sit above the packet's own sequence, its own frame at or below. - if refuse_sequenced && frame.sequence <= packet.sequence { + if refuse_sequenced && sequence_at_or_before(frame.sequence, packet.sequence) { if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = self.refuse_unreliable_write(key, &frame.payload)? { @@ -1816,6 +1829,13 @@ where #[cfg(test)] mod send_failure_tests; +/// Whether `sequence` is at or before `mark` in serial order (RFC 1982): the 32-bit space is compared modulo 2^32, +/// so a frame that began before the wrap and a packet after it still order correctly. The receive window of +/// `cultnet_rs`' duplicate test does not belong here: a frame's first fragment may lie any distance back. +fn sequence_at_or_before(sequence: u32, mark: u32) -> bool { + mark.wrapping_sub(sequence) < 1 << 31 +} + fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.max_sessions == 0 { return Err(anyhow!("max_sessions must be greater than zero")); @@ -1931,4 +1951,15 @@ mod tests { .unwrap(); assert_eq!(hex(&bytes), MESSAGE_INVALID_HEX); } + + #[test] + fn sequence_order_holds_across_the_wrap_and_beyond_the_receive_window() { + assert!(sequence_at_or_before(7, 7)); + assert!(sequence_at_or_before(100, 5_200), "further back than the receive window"); + assert!(!sequence_at_or_before(5_200, 100)); + assert!(sequence_at_or_before(u32::MAX - 1, 1), "begun before the wrap, ended after it"); + assert!(!sequence_at_or_before(1, u32::MAX - 1), "after the wrap is later than before it"); + assert!(sequence_at_or_before(u32::MAX, 0), "the wrap itself"); + assert!(!sequence_at_or_before(0, u32::MAX)); + } } diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index b6663396..d9b42abb 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3097,6 +3097,108 @@ fn a_sequenced_packet_releasing_a_held_ordered_write_refuses_only_its_own() -> R Ok(()) } +/// The refusal of a write on a sequenced channel does not depend on the packet being the newest. A reliable-sequenced +/// packet that arrives after a newer one on its channel is superseded: the session delivers no frame for it, yet +/// the packet was acknowledged and will not be resent, so the server answers it from the packet's own payload. +#[test] +fn a_superseded_sequenced_write_is_still_refused() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 253)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let packet_for = |peer: &mut RawPeer, id: &str| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: false, sequenced: true, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let older = packet_for(&mut peer, "old")?; + let newer = packet_for(&mut peer, "new")?; + let mut refused_ids = Vec::new(); + for packets in [&newer, &older] { + for packet in packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + let answer = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + if matches!(answer, CultNetMessage::ErrorV1 { .. }) { + refused_ids.push(refusal_of(&answer).1.expect("a refusal names its write")); + } + } + } + } + assert_eq!(refused_ids, vec!["new".to_string(), "old".to_string()], "the superseded write is answered too"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "and nothing was applied"); + Ok(()) +} + +/// The refusal reaches a sequenced write however far apart its fragments' sequences sit: other traffic can take the +/// sequences between the first fragment and the last, further back than the duplicate test's receive window. +#[test] +fn a_sequenced_write_whose_fragments_span_more_than_the_receive_window_is_refused() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 254)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let payload = encode_cultnet_message_to_vec(&put("far", "note:far", vec![1; 64]), CultNetWireContract::CultNetSchemaV0)?; + let mut packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: false, sequenced: true, now_ms: 3, reliable_expire_after_ms: None }, + Some(32), + )?; + let last = packets.pop().expect("the write is fragmented"); + assert!(!packets.is_empty()); + let mut next = last.sequence; + let mut refused_ids = Vec::new(); + for packet in &packets { + peer.send_packet(packet)?; + server.poll_once()?; + } + // 5,000 reliable packets on another channel, in order, take the sequences between the fragments. + for count in 0..5_000 { + let mut filler = packets[0].clone(); + filler.channel_id = "filler".into(); + filler.sequenced = false; + filler.fragment_id = 0; + filler.fragment_index = 0; + filler.fragment_count = 0; + filler.payload = vec![0]; + filler.sequence = next; + next += 1; + peer.send_packet(&filler)?; + server.poll_once()?; + if count % 100 == 99 { + // Read what the server sent, so the socket buffer never holds back the refusal. + for packet in peer.drain() { + peer.session.receive(&packet, 4)?; + } + } + } + let mut last = last; + last.sequence = next; + peer.send_packet(&last)?; + server.poll_once()?; + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + let answer = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + if matches!(answer, CultNetMessage::ErrorV1 { .. }) { + refused_ids.push(refusal_of(&answer).1.expect("a refusal names its write")); + } + } + } + assert_eq!(refused_ids, vec!["far".to_string()]); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "and nothing was applied"); + Ok(()) +} + /// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured /// from the refusal and not from the packet before it. #[test] From 34cceedf9841b57f9bc01c7bad5e43199ec7e2ff Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:15:34 +0000 Subject: [PATCH 084/117] Rust RUDP server: pin the superseded-write refusal against a releasing packet and the half-space bound cargo-mutants on the previous commit missed two mutants: `&&` to `||` in the superseded-packet check (a packet that also releases another write hid it) and `<` to `<=` at the half-space bound of sequence_at_or_before. A test and two assertions kill them; the rerun catches 14 of 15 (1 unviable). Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 2 + .../cultmesh-rs/tests/rudp_document_server.rs | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index a0ab51ac..1e729407 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -1961,5 +1961,7 @@ mod tests { assert!(!sequence_at_or_before(1, u32::MAX - 1), "after the wrap is later than before it"); assert!(sequence_at_or_before(u32::MAX, 0), "the wrap itself"); assert!(!sequence_at_or_before(0, u32::MAX)); + assert!(sequence_at_or_before(0, (1 << 31) - 1), "just inside half the space"); + assert!(!sequence_at_or_before(0, 1 << 31), "half the space away is not before"); } } diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index d9b42abb..6ca0007a 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3138,6 +3138,50 @@ fn a_superseded_sequenced_write_is_still_refused() -> Result<()> { Ok(()) } +/// A superseded sequenced packet is answered even when its arrival fills a gap and releases another write: the +/// frames that arrive with it do not stand in for its own. +#[test] +fn a_superseded_sequenced_write_is_refused_while_it_releases_a_held_ordered_write() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 255)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let packet_for = |peer: &mut RawPeer, id: &str, ordered: bool, sequenced: bool| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered, sequenced, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let older = packet_for(&mut peer, "old", false, true)?; + let ordered = packet_for(&mut peer, "o1", true, false)?; + let newer = packet_for(&mut peer, "new", false, true)?; + let mut answers = Vec::new(); + for packets in [&newer, &ordered, &older] { + for packet in packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + } + let mut refused: Vec = answers + .iter() + .filter(|a| matches!(a, CultNetMessage::ErrorV1 { .. })) + .map(|a| refusal_of(a).1.expect("a refusal names its write")) + .collect(); + refused.sort(); + assert_eq!(refused, vec!["new".to_string(), "old".to_string()], "{answers:?}"); + assert!(answers.contains(&admitted("o1", "note:o1")), "the released ordered write is admitted: {answers:?}"); + Ok(()) +} + /// The refusal reaches a sequenced write however far apart its fragments' sequences sit: other traffic can take the /// sequences between the first fragment and the last, further back than the duplicate test's receive window. #[test] From 6c441d1cda7780a90ecbff2dacaa3f0e7ac7e542 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 09:12:59 +0000 Subject: [PATCH 085/117] Rust RUDP server: answer a sequenced write once, not once per send A repeat of a packet the session already took, and a packet too far ahead to be remembered, delivered no frame and so drew the superseded-write refusal again on every send, breaking write-receipt rule 1. The refusal now applies only to the first packet the session takes for a sequence; CultNetRudpSession::has_received exposes that test. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 9 +++ .../cultmesh-rs/tests/rudp_document_server.rs | 58 +++++++++++++++++++ packages/cultnet-rs/src/rudp.rs | 6 ++ 3 files changed, 73 insertions(+) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 1e729407..9aae5be9 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -780,6 +780,13 @@ where return Ok(CultMeshRudpPollOutcome::Handled); } + // Whether this packet is the first the session takes for its sequence: a repeat of an answered write, or a + // packet too far ahead to be remembered, is not, and the write is not answered for it again. + let first_receipt = packet.reliable + && self + .sessions + .get(&key) + .is_some_and(|entry| !entry.session.has_received(packet.sequence)); let result = { let entry = self .sessions @@ -825,6 +832,8 @@ where // go unanswered. A packet that carries its whole frame is answered from its own payload. A superseded // fragment cannot be: the write's id may lie in a fragment this packet does not hold. if refuse_sequenced + && first_receipt + && self.sessions.get(&key).is_some_and(|entry| entry.session.has_received(packet.sequence)) && packet.fragment_count == 0 && !result.delivered.iter().any(|frame| frame.channel_id == "schema" && frame.sequence == packet.sequence) { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 6ca0007a..60596ee5 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3138,6 +3138,64 @@ fn a_superseded_sequenced_write_is_still_refused() -> Result<()> { Ok(()) } +/// One answer per write (write-receipt.md, rule 1): a sequenced packet the server already answered is answered +/// once, however often the peer sends it again, and a packet too far ahead to be received draws nothing until it +/// is received (the peer resends it, and the answer comes then). +#[test] +fn a_repeated_or_far_ahead_sequenced_write_draws_one_answer_not_one_per_send() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 256)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let packet_for = |peer: &mut RawPeer, id: &str| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: false, sequenced: true, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let refusals_of = |peer: &mut RawPeer| -> Result> { + let mut ids = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + let answer = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + if matches!(answer, CultNetMessage::ErrorV1 { .. }) { + ids.push(refusal_of(&answer).1.expect("a refusal names its write")); + } + } + } + Ok(ids) + }; + // The same packet, sent three times (a resend whose acknowledgement was lost), is refused once. + let once = packet_for(&mut peer, "dup")?; + let mut refused = Vec::new(); + for _ in 0..3 { + for packet in &once { + peer.send_packet(packet)?; + } + server.poll_once()?; + refused.extend(refusals_of(&mut peer)?); + } + assert_eq!(refused, vec!["dup".to_string()], "a repeat of an answered write is not answered again"); + + // A packet far past the receive window is not received: no answer yet, and none per resend. + let ahead = packet_for(&mut peer, "ahead")?; + let mut far = ahead[0].clone(); + far.sequence = once[0].sequence + 5_000; + let mut refused = Vec::new(); + for _ in 0..3 { + peer.send_packet(&far)?; + server.poll_once()?; + refused.extend(refusals_of(&mut peer)?); + } + assert_eq!(refused, Vec::::new(), "an unreceived packet draws no answer"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + Ok(()) +} + /// A superseded sequenced packet is answered even when its arrival fills a gap and releases another write: the /// frames that arrive with it do not stand in for its own. #[test] diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index eae7a2ee..427a38c1 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -1277,6 +1277,12 @@ impl CultNetRudpSession { } } + /// Whether `receive` has taken this reliable sequence: delivered, held, superseded or a duplicate. A packet + /// `receive` refused unremembered (too far ahead, or before the handshake) is not, and its sender resends it. + pub fn has_received(&self, sequence: u32) -> bool { + self.was_received(sequence) + } + /// The one duplicate test: true for a sequence in the window that was /// received, for one below the window, and for one at or before the /// watermark. The watermark starts at the handshake's seed, so a frame the From 636b387cf10550fcb335344f679f3f1879d6e8c8 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 09:13:31 +0000 Subject: [PATCH 086/117] write-receipt spec: a repeated or unreceived sequenced packet draws no second answer Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index b74411ea..234fdf93 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -38,7 +38,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. + - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. A repeat of a packet the server already took, or a packet too far ahead to be taken, draws no further answer (rule 1). - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. From b082101c784f44a9910222f95f56ffe4cfb38ee9 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 10:22:15 +0000 Subject: [PATCH 087/117] cultnet-ts: a blank text in a raw frame is Unicode White_Space, not trim() validateBinaryMessageRoot, requireNonEmptyString and requireOptionalStringArray used String.prototype.trim(), which takes U+FEFF and misses U+0085, so the TS decoder and the servers disagreed on what names nothing (write-receipt rule 5). Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 11 ++++++++--- .../cultnet-ts/test/write-receipt.test.ts | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 6e05c9a5..3cc87dcd 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -1422,7 +1422,7 @@ function validateBinaryMessageRoot( if (candidate.schemaVersion !== schemaVersion) { throw new Error(`Validation failed for ${schemaVersion}: /schemaVersion: must equal ${schemaVersion}`); } - if (typeof candidate.messageId !== "string" || candidate.messageId.trim().length === 0) { + if (typeof candidate.messageId !== "string" || isBlank(candidate.messageId)) { throw new Error(`Validation failed for ${schemaVersion}: /messageId: must be a non-empty string`); } @@ -1460,8 +1460,13 @@ function validateRawDocumentRecord( requireOptionalStringArray(candidate.tags, `${instancePath}/tags`); } +/** Blank is empty or only Unicode White_Space (write-receipt.md, rule 5), not `trim()`: JS trims U+FEFF and keeps U+0085. */ +function isBlank(value: string): boolean { + return /^\p{White_Space}*$/u.test(value); +} + function requireNonEmptyString(input: unknown, instancePath: string): void { - if (typeof input !== "string" || input.trim().length === 0) { + if (typeof input !== "string" || isBlank(input)) { throw new Error(`Validation failed for raw CultNet document record: ${instancePath}: must be a non-empty string`); } } @@ -1477,7 +1482,7 @@ function requireOptionalStringArray(input: unknown, instancePath: string): void if (typeof input === "undefined" || input === null) { return; } - if (!Array.isArray(input) || input.some((item) => typeof item !== "string" || item.trim().length === 0)) { + if (!Array.isArray(input) || input.some((item) => typeof item !== "string" || isBlank(item))) { throw new Error(`Validation failed for raw CultNet document record: ${instancePath}: must be an array of non-empty strings`); } } diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 83c31411..efab1c28 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -227,3 +227,22 @@ test("a write whose own send failed settles local_send_failed even when its wind tracker.sendFailed("w1", 1000); assert.deepEqual(tracker.state("w1"), { outcome: "unconfirmed", reason: "local_send_failed" }); }); + +// Blank is Unicode White_Space (write-receipt.md, rule 5), not what String.prototype.trim() trims: trim() takes U+FEFF and +// misses U+0085. Every frame decoder that rejects a blank text must agree with the servers that do. +test("a blank messageId, record key or tag is exactly Unicode White_Space, not trim()", () => { + const put = (document: Record, messageId = "p1") => ({ + schemaVersion: "cultnet.document_put_raw.v0", messageId, + document: { schemaId: "note.v0", recordKey: "note:1", storedAt: "2026-11-08T00:00:00Z", payloadEncoding: "messagepack", payload: Uint8Array.of(0x81, 0xa1, 0x61, 0x01), ...document }, + }); + for (const blank of ["\u0085", "  ", "　"]) { + assert.throws(() => parseCultNetMessage(put({}, blank)), `messageId ${JSON.stringify(blank)}`); + assert.throws(() => parseCultNetMessage(put({ recordKey: blank })), `recordKey ${JSON.stringify(blank)}`); + assert.throws(() => parseCultNetMessage(put({ tags: [blank] })), `tag ${JSON.stringify(blank)}`); + } + for (const named of ["", "​", "\u001c", "᠎"]) { + parseCultNetMessage(put({}, named)); + parseCultNetMessage(put({ recordKey: named })); + parseCultNetMessage(put({ tags: [named] })); + } +}); From f0c0f936ba037504d09960d2d83ab9516074ad06 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 10:34:49 +0000 Subject: [PATCH 088/117] write-receipt: seven tracker vectors, a tracker that ends with its session, runners that count New scenario vectors: id-less error.v1 with one and with two writes unanswered, sendFailed, an id-less error after the window, a receipt one ms inside the window and at the exact boundary (late), a send after sessionEnd. The last exposed a TS tracker leaving such a write pending forever; it now settles session_ended. The file declares counts, and the TS replay runner fails on a count it did not run or an event key scenarios.semantics does not name. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 269 +++++++++++++++++- contracts/cultnet/interop/write-receipt.md | 4 +- packages/cultnet-ts/CHANGELOG.md | 1 + packages/cultnet-ts/src/write-tracker.ts | 10 +- .../cultnet-ts/test/write-receipt.test.ts | 34 +++ .../tools/replay-write-receipt-vectors.mjs | 16 ++ 6 files changed, 329 insertions(+), 5 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 50e8a20f..397fbad6 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -254,7 +254,7 @@ } ], "scenarios": { - "semantics": "Each scenario replays events in atMs order against a client that sent each write at its send event; a tick advances the clock without a frame. 'window' names the constant in vectors.constants that applies (server receipt -> RECEIPT_RESULT_WINDOW_MS, legacy -> LEGACY_RESULT_WINDOW_MS); a tick at or after the window settles unanswered writes as unconfirmed. expect gives the outcome the caller was told per write id; recorded, when present, is the write handle's later state. admitted/refused/unconfirmed are three distinct results; no scenario may settle admitted without a write_admitted frame carrying the write's id.", + "semantics": "Each scenario replays events in atMs order against a client that sent each write at its send event; a tick advances the clock without a frame. An event is atMs plus exactly one of: send (a write goes out), frame (a decoded frame arrives), transportAck (delivery, never an answer; the tracker has no input for it), tick (the clock advances), sendFailed (the client's own send of that write failed), sessionEnd (the session ended). Any other key is a defect in the vectors or the runner. A window ends when a write's age reaches it, so an event at exactly sentAt+window finds the write already unconfirmed. 'window' names the constant in vectors.constants that applies (server receipt -> RECEIPT_RESULT_WINDOW_MS, legacy -> LEGACY_RESULT_WINDOW_MS); a tick at or after the window settles unanswered writes as unconfirmed. expect gives the outcome the caller was told per write id; recorded, when present, is the write handle's later state. admitted/refused/unconfirmed are three distinct results; no scenario may settle admitted without a write_admitted frame carrying the write's id.", "cases": [ { "name": "receipt_admits", @@ -909,6 +909,266 @@ "outcome": "admitted" } } + }, + { + "name": "error_v1_without_id_with_one_write_in_flight_refuses", + "description": "An error.v1 whose messageId is nil names no write: like error.v0 it is attributable only because one write is unanswered.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 30, + "frame": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid", + "routingHint": null, + "code": "message_invalid", + "details": null, + "messageId": null + } + } + ], + "expect": { + "w1": { + "outcome": "refused", + "code": "message_invalid" + } + } + }, + { + "name": "error_v1_without_id_with_two_writes_in_flight_is_unattributed", + "description": "An error.v1 with a nil messageId settles neither of two unanswered writes; each ends unconfirmed at its window.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1", + "w2" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 1, + "send": "w2" + }, + { + "atMs": 30, + "frame": { + "schemaVersion": "cultnet.error.v1", + "error": "message_invalid", + "routingHint": null, + "code": "message_invalid", + "details": null, + "messageId": null + } + }, + { + "atMs": 5001, + "tick": true + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + }, + "w2": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "send_failed_is_unconfirmed_and_touches_no_other_write", + "description": "The client's own send of w1 failed: w1 is unconfirmed local_send_failed, never refused; w2 is unaffected and settles on its own receipt.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1", + "w2" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 1, + "send": "w2" + }, + { + "atMs": 2, + "sendFailed": "w1" + }, + { + "atMs": 20, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w2", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "local_send_failed" + }, + "w2": { + "outcome": "admitted" + } + } + }, + { + "name": "error_without_id_after_the_window_settles_nothing", + "description": "Once w1 is unconfirmed no write is unanswered, so an error that names no write is attributed to nothing and records nothing on w1.", + "server": "legacy", + "window": "LEGACY_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 500, + "tick": true + }, + { + "atMs": 600, + "frame": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": null + } + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed" + } + } + }, + { + "name": "receipt_one_millisecond_inside_the_window_admits", + "description": "A receipt 1 ms before the window ends settles the write admitted.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 4999, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "admitted" + } + } + }, + { + "name": "receipt_at_the_exact_window_boundary_is_late", + "description": "The window ends when the write's age reaches it: a receipt at exactly that instant finds w1 already unconfirmed and is recorded admitted_late.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5000, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + } + ], + "expect": { + "w1": { + "outcome": "unconfirmed", + "reason": "window_elapsed", + "recorded": "admitted_late" + } + } + }, + { + "name": "send_after_session_end_is_unconfirmed_at_once", + "description": "A write registered after the session ended can never be answered: it settles unconfirmed session_ended when sent, and the earlier write keeps its own receipt.", + "server": "receipt", + "window": "RECEIPT_RESULT_WINDOW_MS", + "writes": [ + "w1", + "w2" + ], + "events": [ + { + "atMs": 0, + "send": "w1" + }, + { + "atMs": 5, + "frame": { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "w1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1" + } + }, + { + "atMs": 40, + "sessionEnd": true + }, + { + "atMs": 50, + "send": "w2" + } + ], + "expect": { + "w1": { + "outcome": "admitted" + }, + "w2": { + "outcome": "unconfirmed", + "reason": "session_ended" + } + } } ] }, @@ -917,6 +1177,13 @@ "LEGACY_RESULT_WINDOW_MS": 500, "DEDUPE_RETENTION_MS": 600000 }, + "counts": { + "semantics": "How many of each a runner must have run. A runner that reads fewer (a parser that drops entries, a loop over nothing) fails instead of passing quietly; a runner that meets an event key outside scenarios.semantics fails too.", + "wire": 7, + "schemaInvalid": 7, + "contentHashCases": 4, + "scenarioCases": 25 + }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", "cases": [ diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index d3dc2614..b01e69fb 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -49,9 +49,9 @@ A write settles as exactly one of: - **admitted**: a `write_admitted` whose `messageId` is the write's. - **refused**: an `error.v1` whose `messageId` is the write's, carrying its `code` and `details` as the typed error. -- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window), `session_ended` (the peer closed the session, or it ended, before the write was answered) and `local_send_failed` (the client's own send of this write failed). A local send failure is never reported as refused: no server answer exists, and the write may or may not have left the machine. Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. +- **unconfirmed**: nothing settled it. Reasons: `window_elapsed` (the result window ran from the *send* and passed with no answer; see Result window), `session_ended` (the peer closed the session, or it ended, before the write was answered; a write registered after the session ended settles this way when it is sent) and `local_send_failed` (the client's own send of this write failed). A local send failure is never reported as refused: no server answer exists, and the write may or may not have left the machine. Unconfirmed is not success and not refusal. APIs that return a value return the tri-state; APIs that throw throw a distinct refused error and a distinct unconfirmed error and return normally only for admitted. A caller that needs admission treats unconfirmed as not admitted. Resend with the same `messageId` and unchanged content, **on the same session**, is safe against a receipt-capable server (rule 5). After a reconnect the resend is a new write, and a legacy server may apply a resend twice: the caller resends only writes that are idempotent by record key (a put of the same content to the same key, a delete of a key), and reads the record back before resending any other. -**Result window.** Two named constants (also in the vectors): `RECEIPT_RESULT_WINDOW_MS = 5000`, the default against a receipt-capable server (its hello lists both frames); `LEGACY_RESULT_WINDOW_MS = 500`, used only as the fallback against a server whose hello does not list them (or that sent none). The window runs from the send. A receipt-capable server that has not answered within its window is reported as unconfirmed like any other; clients log it as a server that missed its window. +**Result window.** Two named constants (also in the vectors): `RECEIPT_RESULT_WINDOW_MS = 5000`, the default against a receipt-capable server (its hello lists both frames); `LEGACY_RESULT_WINDOW_MS = 500`, used only as the fallback against a server whose hello does not list them (or that sent none). The window runs from the send. It ends when the write's age reaches it: an answer arriving at exactly that instant is late. A receipt-capable server that has not answered within its window is reported as unconfirmed like any other; clients log it as a server that missed its window. **Late answers.** An answer for a write that already settled `unconfirmed` does not change what the caller was told, but the client records it on the write: a late `write_admitted` makes the write's recorded state `admitted_late`, a late refusal makes it `refused_late`. The write handle exposes that state, and a caller that got `unconfirmed` MUST consult it (and the record) before resending: it must not blind-resend a write that was admitted late. A resend on the same session with the same id and content is harmless on a receipt-capable server (rule 5); across a reconnect the caller follows the idempotence rule above. diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index be6e2d98..e7a5f1f1 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@gamecult/cultnet-ts` are recorded here, following ### Added +- `WriteTracker`: a write sent after `sessionEnd` settles unconfirmed `session_ended` when it is sent (it stayed unanswered forever). The replay runner now fails on an event key outside `scenarios.semantics` and on a vector count that differs from the file's `counts`; seven tracker vectors are new. A blank text in a raw frame is Unicode White_Space, not `String.prototype.trim()`. - Write receipt, client side (`contracts/cultnet/interop/write-receipt.md`): `cultnet.write_admitted.v0` and `cultnet.error.v1` (v0 plus `messageId`; its closed code set adds `limit_reached`, `CULTNET_ERROR_V1_CODES`) parse and encode, and `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS` / `helloAdvertisesWriteReceipts` name the hello capability. `cultnet.error.v0` and its code set are unchanged. `WriteTracker` settles each write as admitted, refused or unconfirmed (windows `RECEIPT_RESULT_WINDOW_MS` 5000 and `LEGACY_RESULT_WINDOW_MS` 500, counted from the send; late answers recorded as `admitted_late` / `refused_late`), and `tools/replay-write-receipt-vectors.mjs` replays the vectors against it (see README, "Write receipt replay runner"). The interop schemas are read from `contracts/cultnet/interop/`. - `message_invalid` joins the closed `cultnet.error.v0` code set, with `cultNetMessageInvalidError(field, reason)` (`details.field` names the request field at fault), byte-identical to the C# `ForMessageInvalid`. - `cultNetWriteNotAuthorizedError(recordKey, reason?)`: the typed `write_not_authorized` refusal (code and `details.recordKey`), byte-identical to the C# `ForRemoteWriteRefused` vector in `error-vectors.json`. `CultNetDocumentRegistry.canonicalSchemaId(schemaId)` resolves the id a put or delete writes under, for a server's write authority. diff --git a/packages/cultnet-ts/src/write-tracker.ts b/packages/cultnet-ts/src/write-tracker.ts index 2204d371..cedf877d 100644 --- a/packages/cultnet-ts/src/write-tracker.ts +++ b/packages/cultnet-ts/src/write-tracker.ts @@ -49,6 +49,7 @@ export class WriteTracker { readonly #windowMs: number; readonly #onSettle?: (messageId: string, settlement: WriteSettlement) => void; readonly #writes = new Map(); + #ended = false; constructor(options: WriteTrackerOptions) { this.#windowMs = options.windowMs; @@ -58,7 +59,11 @@ export class WriteTracker { /** Registers a write sent at `atMs`. A messageId is minted once per session: re-sending a known id is the same write. */ send(messageId: string, atMs: number): void { this.tick(atMs); - if (!this.#writes.has(messageId)) this.#writes.set(messageId, { sentAtMs: atMs, state: { outcome: "pending" } }); + if (this.#writes.has(messageId)) return; + const entry: Entry = { sentAtMs: atMs, state: { outcome: "pending" } }; + this.#writes.set(messageId, entry); + // On a session that already ended nothing can answer this write. + if (this.#ended) this.#settle(messageId, entry, { outcome: "unconfirmed", reason: "session_ended" }); } /** Feeds one decoded frame received at `atMs`. Frames that are not answers to a write are ignored. */ @@ -93,9 +98,10 @@ export class WriteTracker { } } - /** The session ended: every unanswered write settles unconfirmed. */ + /** The session ended: every unanswered write settles unconfirmed, and so does any write sent afterwards. */ sessionEnd(atMs: number): void { this.tick(atMs); + this.#ended = true; for (const [messageId, entry] of this.#writes) { if (entry.state.outcome === "pending") this.#settle(messageId, entry, { outcome: "unconfirmed", reason: "session_ended" }); } diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index efab1c28..6acb79cf 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -1,6 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -195,6 +197,38 @@ test("the replay runner passes every vector against this runtime", () => { assert.doesNotMatch(run.stdout, /^FAIL/m); }); +// A runner that skips what it cannot read passes a file it did not replay. It must fail instead. +test("the replay runner fails on an unknown event key and on a vector count the file does not declare", () => { + const script = join(__dirname, "..", "..", "tools", "replay-write-receipt-vectors.mjs"); + const vectorsPath = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + const dir = mkdtempSync(join(tmpdir(), "receipt-vectors-")); + const runWith = (edit: (vectors: any) => void) => { + const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); + edit(vectors); + const file = join(dir, "vectors.json"); + writeFileSync(file, JSON.stringify(vectors)); + return spawnSync(process.execPath, [script, file], { encoding: "utf8" }); + }; + const typo = runWith((v) => { v.scenarios.cases[0].events[0] = { atMs: 0, sendfailed: "w1" }; }); + assert.equal(typo.status, 1, typo.stdout); + assert.match(typo.stdout, /^FAIL scenario\/receipt_admits$/m); + const dropped = runWith((v) => { v.scenarios.cases.pop(); }); + assert.equal(dropped.status, 1, dropped.stdout); + assert.match(dropped.stdout, /^FAIL counts$/m); + const undeclared = runWith((v) => { v.wire.push(v.wire[0]); }); + assert.equal(undeclared.status, 1, undeclared.stdout); +}); + +test("a write sent after the session ended settles unconfirmed session_ended at once, and the earlier receipt stands", () => { + const settled: [string, WriteSettlement][] = []; + const tracker = new WriteTracker({ windowMs: 1000, onSettle: (id, settlement) => settled.push([id, settlement]) }); + tracker.send("w1", 0); + tracker.frame(admitted("w1"), 5); + tracker.sessionEnd(40); + tracker.send("w2", 50); + assert.deepEqual(settled, [["w1", { outcome: "admitted" }], ["w2", { outcome: "unconfirmed", reason: "session_ended" }]]); +}); + test("a failed local send settles only that write unconfirmed as local_send_failed, and an answer after it is recorded late", () => { const settled: [string, WriteSettlement][] = []; const tracker = new WriteTracker({ windowMs: 1000, onSettle: (id, settlement) => settled.push([id, settlement]) }); diff --git a/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs b/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs index 2c4bb403..87da14e9 100644 --- a/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs +++ b/packages/cultnet-ts/tools/replay-write-receipt-vectors.mjs @@ -23,6 +23,9 @@ const { const vectorsPath = process.argv[2] ?? path.join(here, "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); +// The event actions scenarios.semantics names; anything else is refused, not skipped. +const EVENT_KEYS = ["send", "frame", "tick", "sessionEnd", "sendFailed", "transportAck"]; + const results = []; const check = (name, run) => { try { @@ -79,10 +82,13 @@ for (const scenario of vectors.scenarios.cases) { told.set(id, settlement); } }); for (const event of [...scenario.events].sort((a, b) => a.atMs - b.atMs)) { + const actions = Object.keys(event).filter((key) => key !== "atMs"); + if (actions.length !== 1 || !EVENT_KEYS.includes(actions[0])) { problems.push(`event at ${event.atMs} has keys [${actions}], wanted atMs and exactly one of ${EVENT_KEYS}`); continue; } if ("send" in event) tracker.send(event.send, event.atMs); else if ("frame" in event) tracker.frame(delivered(event.frame), event.atMs); else if ("tick" in event) tracker.tick(event.atMs); else if ("sessionEnd" in event) tracker.sessionEnd(event.atMs); + else if ("sendFailed" in event) tracker.sendFailed(event.sendFailed, event.atMs); // transportAck: delivery is not an answer; the tracker has no such input and must not need one. } for (const [id, want] of Object.entries(scenario.expect)) { @@ -99,6 +105,16 @@ for (const scenario of vectors.scenarios.cases) { }); } +// A runner that read fewer vectors than the file declares has silently skipped some. (contentHash is not replayed here.) +check("counts", () => { + const ran = (prefix) => results.filter((r) => r.name.startsWith(prefix)).length; + const loaded = { wire: vectors.wire.length, schemaInvalid: vectors.schemaInvalid.length, scenarioCases: vectors.scenarios.cases.length }; + const executed = { wire: ran("wire/"), schemaInvalid: ran("schemaInvalid/"), scenarioCases: ran("scenario/") }; + return Object.keys(executed) + .filter((key) => loaded[key] !== vectors.counts[key] || executed[key] !== vectors.counts[key]) + .map((key) => `${key}: declared ${vectors.counts[key]}, loaded ${loaded[key]}, ran ${executed[key]}`); +}); + function sorted(value) { if (Array.isArray(value)) return value.map(sorted); if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((k) => [k, sorted(value[k])])); From 6ca9c455b71c07591603e7b07fd08dfc762dae7b Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 10:37:48 +0000 Subject: [PATCH 089/117] cultmesh-kotlin: sendFailed vectors, a tracker that ends with its session, White_Space blank The replay runner handles the sendFailed event, fails on an event key it does not know and on a vector count the file did not declare. WriteTracker settles a write sent after sessionEnd as session_ended. Source ids on a raw document record are dropped when blank by Unicode White_Space (isBlankText), not Kotlin's isNullOrBlank, which keeps U+0085 and drops U+001C to U+001F; the vector file's JSON reader skips JSON whitespace only. Co-Authored-By: Claude Sonnet 5.5 --- .../kotlin/org/gamecult/cultmesh/CultMesh.kt | 7 ++--- .../org/gamecult/cultmesh/WriteReceipt.kt | 27 ++++++++++++++++++- .../gamecult/cultmesh/WriteReceiptVectors.kt | 21 ++++++++++++++- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index d9cdfad7..4a06d7f5 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -1100,9 +1100,9 @@ data class CultNetRawDocumentRecord( "payloadEncoding" to payloadEncoding, "payload" to payload.copyOf(), ) - if (!sourceRuntimeId.isNullOrBlank()) wire["sourceRuntimeId"] = sourceRuntimeId - if (!sourceAgentId.isNullOrBlank()) wire["sourceAgentId"] = sourceAgentId - if (!sourceRole.isNullOrBlank()) wire["sourceRole"] = sourceRole + if (!isBlankText(sourceRuntimeId)) wire["sourceRuntimeId"] = sourceRuntimeId + if (!isBlankText(sourceAgentId)) wire["sourceAgentId"] = sourceAgentId + if (!isBlankText(sourceRole)) wire["sourceRole"] = sourceRole if (tags.isNotEmpty()) wire["tags"] = tags return wire } @@ -3413,6 +3413,7 @@ fun main(args: Array) { rudpClientBindsLoopbackOnlyForALoopbackEndpoint() rudpSocketTransportHandshakesAndCarriesReliableOrderedSchemaFrames() rudpSocketTransportCarriesFragmentedReliableOrderedSchemaFrames() + rawDocumentRecordDropsOnlyWhiteSpaceSourceIds() return } diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt index 4060f08a..4b38d942 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -163,11 +163,16 @@ class WriteTracker(private val windowMs: Long, private val onSettle: (String, Wr } private val writes = linkedMapOf() + private var ended = false /** Registers a write sent at [atMs]. Re-sending a known id is the same write and does not restart its window. */ fun send(messageId: String, atMs: Long) { tick(atMs) - writes.getOrPut(messageId) { Entry(atMs) } + if (messageId in writes) return + val entry = Entry(atMs) + writes[messageId] = entry + // On a session that already ended nothing can answer this write. + if (ended) settle(messageId, entry, WriteSettlement.Unconfirmed(WriteUnconfirmedReason.SESSION_ENDED)) } /** Feeds one received message. Only write_admitted, error.v1 and error.v0 answer; anything else is ignored. */ @@ -200,8 +205,10 @@ class WriteTracker(private val windowMs: Long, private val onSettle: (String, Wr } } + /** The session ended: every unanswered write settles unconfirmed, and so does any write sent afterwards. */ fun sessionEnd(atMs: Long) { tick(atMs) + ended = true for ((id, entry) in writes) if (entry.settlement == null) settle(id, entry, WriteSettlement.Unconfirmed(WriteUnconfirmedReason.SESSION_ENDED)) } @@ -230,3 +237,21 @@ class WriteTracker(private val windowMs: Long, private val onSettle: (String, Wr onSettle(messageId, settlement) } } + +/** Unicode White_Space (PropList), the one definition of "blank" in write-receipt.md rule 5; Kotlin's own isBlank takes U+001C to U+001F. */ +private val WHITE_SPACE_CODE_POINTS: Set = (0x0009..0x000D).toSet() + 0x0020 + 0x0085 + 0x00A0 + 0x1680 + + (0x2000..0x200A).toSet() + 0x2028 + 0x2029 + 0x202F + 0x205F + 0x3000 + +/** True for null, empty, or only White_Space code points. */ +fun isBlankText(text: String?): Boolean = text == null || text.codePoints().allMatch { it in WHITE_SPACE_CODE_POINTS } + +/** Self-test: blank is White_Space, not Kotlin's isBlank (rule 5): U+001C to U+001F name something, U+00A0 and U+0085 do not. */ +fun rawDocumentRecordDropsOnlyWhiteSpaceSourceIds() { + fun wireKeys(id: String) = CultNetRawDocumentRecord("note.v0", "note:1", "2026-11-08T00:00:00Z", payload = byteArrayOf(1), sourceRuntimeId = id, sourceAgentId = id, sourceRole = id).toWireMap().keys + for (blank in listOf("", "\u0085", "\u00a0\u2003", "\u3000", "\t\r\n ")) { + check(isBlankText(blank) && "sourceRuntimeId" !in wireKeys(blank)) { "blank ${blank.map { it.code }} must be dropped" } + } + for (named in listOf("\u001c", "\u001f", "\u200b", "\ufeff", "\u180e", "a")) { + check(!isBlankText(named) && "sourceRuntimeId" in wireKeys(named) && "sourceRole" in wireKeys(named)) { "${named.map { it.code }} names something and must be kept" } + } +} diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index 033ef8fd..e527692d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -8,6 +8,9 @@ package org.gamecult.cultmesh import java.io.File import java.io.IOException +// The event actions scenarios.semantics names; anything else is refused, not skipped. +private val EVENT_KEYS = listOf("send", "frame", "tick", "sessionEnd", "sendFailed", "transportAck") + fun replayWriteReceiptVectors(path: String): Boolean { @Suppress("UNCHECKED_CAST") val vectors = JsonReader(File(path).readText(Charsets.UTF_8)).readDocument() as Map @@ -87,12 +90,15 @@ fun replayWriteReceiptVectors(path: String): Boolean { } for (event in list(scenario["events"]).map(::obj).sortedBy { it["atMs"] as Long }) { val at = event["atMs"] as Long + val actions = event.keys - "atMs" + if (actions.size != 1 || actions.first() !in EVENT_KEYS) { problems += "event at $at has keys $actions, wanted atMs and exactly one of $EVENT_KEYS"; continue } when { "send" in event -> tracker.send(event["send"] as String, at) // A frame reaches the client as bytes: the fake server encodes it, the client decodes it. "frame" in event -> tracker.frame(parseCultNetMessage(MessagePackWriter().value(event["frame"]).toByteArray()), at) "tick" in event -> tracker.tick(at) "sessionEnd" in event -> tracker.sessionEnd(at) + "sendFailed" in event -> tracker.sendFailed(event["sendFailed"] as String, at) // transportAck: delivery is not an answer; the tracker has no such input and must not need one. } } @@ -116,6 +122,19 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // A runner that read fewer vectors than the file declares has silently skipped some. + check("counts") { + val declared = obj(vectors["counts"]) + fun ran(prefix: String) = results.keys.count { it.startsWith(prefix) } + val loaded = mapOf( + "wire" to list(vectors["wire"]).size, "schemaInvalid" to list(vectors["schemaInvalid"]).size, + "contentHashCases" to list(obj(vectors["contentHash"])["cases"]).size, "scenarioCases" to list(obj(vectors["scenarios"])["cases"]).size, + ) + val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/")) + loaded.keys.filter { loaded[it] != (declared[it] as Long).toInt() || executed[it] != loaded[it] } + .map { "$it: declared ${declared[it]}, loaded ${loaded[it]}, ran ${executed[it]}" } + } + for ((name, problems) in results) { println(if (problems.isEmpty()) "PASS $name" else "FAIL $name") problems.forEach { println(" $it") } @@ -144,7 +163,7 @@ private class JsonReader(private val text: String) { fun readDocument(): Any? = readValue().also { skipSpace(); if (at != text.length) throw IOException("trailing JSON") } - private fun skipSpace() { while (at < text.length && text[at].isWhitespace()) at++ } + private fun skipSpace() { while (at < text.length && text[at] in " \t\r\n") at++ } private fun readValue(): Any? { skipSpace() From 8c31933e108b00240708a32a9b9db12a4644e0c3 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 10:38:29 +0000 Subject: [PATCH 090/117] cultnet-py: the write-receipt vector sections include the file's counts Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-py/tests/test_write_receipt.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 32c5f03a..98e74287 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -19,7 +19,7 @@ from cultnet_py import ( ) # Decoder-side conformance with contracts/cultnet/interop/write-receipt-vectors.json: `contentHash`, `wire` and -# `schemaInvalid`. `dedupe` and `blankFields` are server obligations and `scenarios` is the client write tracker; +# `schemaInvalid`. `dedupe` and `blankFields` are server obligations and `scenarios` is the client write tracker; `counts` is the file's own tally; # this runtime has neither a receipt server nor a write client, so it replays none of them. _INTEROP = Path(__file__).resolve().parents[3] / "contracts/cultnet/interop" _VECTORS = json.loads((_INTEROP / "write-receipt-vectors.json").read_text(encoding="utf-8")) @@ -57,11 +57,17 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) self.assertEqual(len(_VECTORS["schemaInvalid"]), 7) + # The file declares how many of each it holds; a loader that drops entries disagrees with it. + counts = _VECTORS["counts"] + self.assertEqual(counts["wire"], len(_VECTORS["wire"])) + self.assertEqual(counts["schemaInvalid"], len(_VECTORS["schemaInvalid"])) + self.assertEqual(counts["contentHashCases"], len(_VECTORS["contentHash"]["cases"])) + self.assertEqual(counts["scenarioCases"], len(_VECTORS["scenarios"]["cases"])) def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: From 1704fb5060f392395c9df55522bcf45f943c2f6e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:05:57 +0000 Subject: [PATCH 091/117] cultmesh-ts: test:interop builds cultcache-ts and cultnet-ts (with contracts) first Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index 0fef7419..39e0abba 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -25,7 +25,7 @@ "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --expose-gc --test dist-test/test/*.test.js", - "test:interop": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/write-receipt-interop.test.js" + "test:interop": "npm --prefix ../.. run build --workspace @gamecult/cultcache-ts --workspace @gamecult/cultnet-ts && node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/write-receipt-interop.test.js" }, "keywords": [ "cultmesh", From b8e40dc372660774535f021c39a70aa7b968679e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:10:42 +0000 Subject: [PATCH 092/117] interop test: pace C# relay connections under the server's per-IP per-minute limit Co-Authored-By: Claude Sonnet 5.5 --- .../test/interop/write-receipt-interop.test.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts index 12bd4394..092d5ffb 100644 --- a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts +++ b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts @@ -16,7 +16,7 @@ import { execFile, spawn, spawnSync, type ChildProcess } from "node:child_proces import { readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { Duplex } from "node:stream"; -import { after, before, describe, test } from "node:test"; +import { after, before, beforeEach, describe, test } from "node:test"; import { promisify } from "node:util"; import { encode } from "@msgpack/msgpack"; import { z } from "zod"; @@ -134,6 +134,20 @@ const rust: Target = { }, }; +// The C# server admits 30 connection requests per IP per minute (Server.MaxConnectionAttemptsPerMinute) and answers the rest with +// no hello. Every scenario wants a fresh session, so before each C# test the harness waits until the last minute holds at most +// CONNECTIONS_PER_MINUTE connections, leaving room for the two a test may open. The server's limit stays as it is. +const CONNECTIONS_PER_MINUTE = 24; +const connectedAt: number[] = []; +async function awaitConnectionBudget(): Promise { + for (;;) { + const now = Date.now(); + while (connectedAt.length > 0 && now - connectedAt[0] >= 61_000) connectedAt.shift(); + if (connectedAt.length <= CONNECTIONS_PER_MINUTE) return; + await sleep(connectedAt[0] + 61_000 - now); + } +} + const csharp: Target = { name: "csharp", shard: { shardId: "receipt-interop-shard", shardEpoch: 1 }, @@ -143,6 +157,7 @@ const csharp: Target = { return launch(dotnet, [csharpDll, "receipt-serve"]); }, async connect(server) { + connectedAt.push(Date.now()); const relay = spawn(dotnet, [csharpDll, "receipt-relay", "--port", String(server.port)], { stdio: ["pipe", "pipe", "ignore"] }); const peer = new CultNetPeer(Duplex.from({ readable: relay.stdout!, writable: relay.stdin! }), { wireContract: "cultnet.schema.v0" }); return { peer, close: () => { relay.stdin!.end(); relay.kill(); } }; @@ -235,6 +250,7 @@ for (const target of [rust, csharp]) { let session: Session | undefined; before(async () => { server = await target.start(); }, { timeout: 180_000 }); after(async () => { session?.close(); await server?.stop(); }); + beforeEach(async () => { if (target === csharp) await awaitConnectionBudget(); }, { timeout: 90_000 }); /** A fresh receipt session (hello sent, the server's hello read) for a scenario. */ async function receiptSession(): Promise { From 96c5fb1ca4dfa1e4523201573af49cb108082a00 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:12:38 +0000 Subject: [PATCH 093/117] interop peer: C# store holds the spec's admitted schema-id names (compatible ids) Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs index 75eaf4da..6871ea00 100644 --- a/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs +++ b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs @@ -14,7 +14,9 @@ using R3; // ordered payload, and on the pipe it is a 4-byte big-endian length followed by the frame. namespace GameCult.Networking.InteropPeer { - [CultDocument("receipt.interop_note", ReceiptHarness.SchemaId)] + // CompatibleSchemaIds are the names the spec's blankFields vectors admit (U+200B, U+001C, and a name padded with NBSP and EM SPACE): + // a name is any string that is not blank, so the store must hold a record put under one of them. + [CultDocument("receipt.interop_note", ReceiptHarness.SchemaId, CompatibleSchemaIds = ["\u200b", "\u001c", "\u00a0a\u2003"])] [MessagePack.MessagePackObject] public sealed class ReceiptInteropNote { From 4571a0d4e7f1e9b0da9ddfec962c0dc7a7fa1366 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:39:51 +0000 Subject: [PATCH 094/117] tests first: write_admitted messageId rule vectors (failing) Pins Heimdall's rule: a write_admitted messageId needs a Unicode letter or number and no Default_Ignorable_Code_Point, compared raw. Shared vectors plus a runner test in TS, Python, Kotlin, C# and Rust. No decoder changed; the refuse vectors fail. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 277 +++++++++++++++++- contracts/cultnet/interop/write-receipt.md | 4 +- .../gamecult/cultmesh/WriteReceiptVectors.kt | 23 +- .../cultnet-py/tests/test_write_receipt.py | 27 +- packages/cultnet-rs/tests/write_receipt.rs | 48 +++ .../cultnet-ts/test/write-receipt.test.ts | 21 ++ .../CultNetWriteAdmittedMessageIdTests.cs | 58 ++++ 7 files changed, 454 insertions(+), 4 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 397fbad6..a37d9660 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -149,6 +149,280 @@ { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" } ] }, + "admittedMessageId": { + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Scope: write_admitted only; error.v1 is untouched.", + "cases": [ + { + "name": "refuse_braille_blank_2800", + "codePoints": [ + "2800" + ], + "expect": "refuse", + "why": "U+2800 is Default_Ignorable and not a letter or number" + }, + { + "name": "refuse_hangul_filler_3164", + "codePoints": [ + "3164" + ], + "expect": "refuse", + "why": "U+3164 Default_Ignorable (a letter by category Lo, so DI decides)" + }, + { + "name": "refuse_hangul_choseong_filler_115F", + "codePoints": [ + "115F" + ], + "expect": "refuse", + "why": "U+115F Default_Ignorable (category Lo)" + }, + { + "name": "refuse_halfwidth_hangul_filler_FFA0", + "codePoints": [ + "FFA0" + ], + "expect": "refuse", + "why": "U+FFA0 Default_Ignorable (category Lo)" + }, + { + "name": "refuse_combining_grapheme_joiner_034F", + "codePoints": [ + "034F" + ], + "expect": "refuse", + "why": "U+034F Default_Ignorable" + }, + { + "name": "refuse_combining_mark_alone_0300", + "codePoints": [ + "0300" + ], + "expect": "refuse", + "why": "a combining mark is not a letter or number" + }, + { + "name": "refuse_private_use_E000", + "codePoints": [ + "E000" + ], + "expect": "refuse", + "why": "private use is neither a letter nor a number" + }, + { + "name": "refuse_unassigned_0378", + "codePoints": [ + "0378" + ], + "expect": "refuse", + "why": "unassigned since Unicode 1.1, neither a letter nor a number" + }, + { + "name": "refuse_lone_surrogate_D800", + "codePoints": [ + "D800" + ], + "expect": "refuse", + "why": "not a Unicode scalar value; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str, so the decoder must refuse the frame. Rust, C#, Kotlin and Python build the frame from wireHex; a runtime whose strings can hold a lone surrogate (TypeScript) may also build it from the code point", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "refuse_control_nul_0000", + "codePoints": [ + "0000" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_bel_0007", + "codePoints": [ + "0007" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_unit_separator_001F", + "codePoints": [ + "001F" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_delete_007F", + "codePoints": [ + "007F" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_next_line_0085", + "codePoints": [ + "0085" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_empty", + "codePoints": [], + "expect": "refuse", + "why": "empty" + }, + { + "name": "refuse_whitespace_only_space_0020", + "codePoints": [ + "0020" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_whitespace_only_nbsp_00A0", + "codePoints": [ + "00A0" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_whitespace_only_ideographic_space_3000", + "codePoints": [ + "3000" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_letter_zwsp_letter", + "codePoints": [ + "0061", + "200B", + "0062" + ], + "expect": "refuse", + "why": "a Default_Ignorable (U+200B) beside letters" + }, + { + "name": "refuse_letter_cgj", + "codePoints": [ + "0061", + "034F" + ], + "expect": "refuse", + "why": "a Default_Ignorable (U+034F) after a letter" + }, + { + "name": "refuse_letter_soft_hyphen", + "codePoints": [ + "0061", + "00AD" + ], + "expect": "refuse", + "why": "U+00AD is Default_Ignorable" + }, + { + "name": "refuse_letter_zwj", + "codePoints": [ + "0061", + "200D" + ], + "expect": "refuse", + "why": "U+200D is Default_Ignorable" + }, + { + "name": "refuse_digit_mongolian_vowel_separator", + "codePoints": [ + "0031", + "180E" + ], + "expect": "refuse", + "why": "U+180E is Default_Ignorable since Unicode 6.3" + }, + { + "name": "refuse_digit_word_joiner_digit", + "codePoints": [ + "0031", + "2060", + "0032" + ], + "expect": "refuse", + "why": "U+2060 is Default_Ignorable" + }, + { + "name": "accept_abc", + "codePoints": [ + "0061", + "0062", + "0063" + ], + "expect": "accept", + "why": "letters" + }, + { + "name": "accept_digit_1", + "codePoints": [ + "0031" + ], + "expect": "accept", + "why": "a number" + }, + { + "name": "accept_padded_a_raw_compare", + "codePoints": [ + "0020", + "0061", + "0020" + ], + "expect": "accept", + "why": "raw compare: the id is echoed byte for byte, never trimmed, so the padding stays and the letter makes it valid" + }, + { + "name": "accept_e_acute_precomposed", + "codePoints": [ + "00E9" + ], + "expect": "accept", + "why": "a letter" + }, + { + "name": "accept_arabic_indic_digit_three", + "codePoints": [ + "0663" + ], + "expect": "accept", + "why": "category Nd, a number" + }, + { + "name": "accept_cjk_zhong", + "codePoints": [ + "4E2D" + ], + "expect": "accept", + "why": "a letter (Lo, not Default_Ignorable)" + }, + { + "name": "accept_letter_then_combining_acute", + "codePoints": [ + "0065", + "0301" + ], + "expect": "accept", + "why": "a combining mark after a letter is allowed" + }, + { + "name": "accept_256_ascii_letters", + "codePoints": [ + "0061" + ], + "expect": "accept", + "why": "256 bytes is the size limit and is allowed", + "repeat": 256 + } + ] + }, "schemaInvalid": [ { "name": "write_admitted_with_an_error_key", @@ -1182,7 +1456,8 @@ "wire": 7, "schemaInvalid": 7, "contentHashCases": 4, - "scenarioCases": 25 + "scenarioCases": 25, + "admittedMessageIdCases": 32 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index dc138452..150c10f8 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,6 +12,8 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. + **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. Why a new error version and not an extra key on v0: the v0 schema is closed (`additionalProperties: false`) and the TypeScript decoder (Ajv over the closed schemas) rejects unknown keys and unknown `schemaVersion`s, and `parseCultNetMessage` throws on an unknown version; Rust's decoder errors on an unknown version too. Unknown-key tolerance of the Rust, Python and Kotlin error decoders was not checked, so none is assumed, so a surprise key or frame breaks an old client's whole receive path. Old clients are therefore never sent either new frame. This reverses the "old clients ignore it" assumption, which is false for today's decoders. @@ -65,7 +67,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; the lone surrogate U+D800 is carried as a whole frame in `wireHex`, because it is not valid UTF-8 and only some runtimes' strings can hold it), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance status diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index e527692d..8955c52d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -48,6 +48,26 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // admittedMessageId: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, compared + // raw. The frame is built with this runtime's own encoder; a case with `wireHex` (the lone surrogate, invalid UTF-8) + // supplies the whole frame. + for (case in list(obj(vectors["admittedMessageId"])["cases"])) { + val c = obj(case) + check("admittedMessageId/${c["name"]}") { + val id = list(c["codePoints"]).joinToString("") { String(Character.toChars((it as String).toInt(16))) }.repeat(((c["repeat"] as Long?) ?: 1L).toInt()) + val bytes = (c["wireHex"] as String?)?.let(::unhex) ?: MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray() + val decoded = try { decodeReceiptFrame(parseCultNetMessage(bytes)) } catch (_: IOException) { null } + when { + c["expect"] == "refuse" -> if (decoded == null) emptyList() else listOf("accepted an id that must be refused (${c["why"]})") + decoded == null -> listOf("refused an id that must be accepted (${c["why"]})") + decoded.toWireMap()["messageId"] != id -> listOf("id not echoed raw: ${decoded.toWireMap()["messageId"]}") + else -> emptyList() + } + } + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { @@ -129,8 +149,9 @@ fun replayWriteReceiptVectors(path: String): Boolean { val loaded = mapOf( "wire" to list(vectors["wire"]).size, "schemaInvalid" to list(vectors["schemaInvalid"]).size, "contentHashCases" to list(obj(vectors["contentHash"])["cases"]).size, "scenarioCases" to list(obj(vectors["scenarios"])["cases"]).size, + "admittedMessageIdCases" to list(obj(vectors["admittedMessageId"])["cases"]).size, ) - val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/")) + val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/"), "admittedMessageIdCases" to ran("admittedMessageId/")) loaded.keys.filter { loaded[it] != (declared[it] as Long).toInt() || executed[it] != loaded[it] } .map { "$it: declared ${declared[it]}, loaded ${loaded[it]}, ran ${executed[it]}" } } diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 98e74287..0c42a6af 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -57,7 +57,7 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) @@ -68,6 +68,31 @@ class WriteReceiptVectorTests(unittest.TestCase): self.assertEqual(counts["schemaInvalid"], len(_VECTORS["schemaInvalid"])) self.assertEqual(counts["contentHashCases"], len(_VECTORS["contentHash"]["cases"])) self.assertEqual(counts["scenarioCases"], len(_VECTORS["scenarios"]["cases"])) + self.assertEqual(counts["admittedMessageIdCases"], len(_VECTORS["admittedMessageId"]["cases"])) + + def test_admitted_message_id_vectors_are_refused_or_accepted_raw(self) -> None: + # `admittedMessageId`: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, + # compared raw (never trimmed). A lone surrogate is not valid UTF-8, so its vector carries the whole frame. + cases = _VECTORS["admittedMessageId"]["cases"] + failures = [] + for case in cases: + raw = ( + bytes.fromhex(case["wireHex"]) + if "wireHex" in case + else _pack(_admitted(messageId="".join(chr(int(cp, 16)) for cp in case["codePoints"]) * case.get("repeat", 1))) + ) + try: + message = parse_message(raw) + except Exception: # noqa: BLE001 - any refusal counts + refused = True + else: + refused = False + wanted = "".join(chr(int(cp, 16)) for cp in case["codePoints"]) * case.get("repeat", 1) + if message.body["messageId"] != wanted: + failures.append(f"{case['name']}: id not echoed raw") + if refused != (case["expect"] == "refuse"): + failures.append(f"{case['name']}: expected {case['expect']}") + self.assertEqual(failures, []) def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 3d233a61..8dccf996 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -153,3 +153,51 @@ fn error_v0_refuses_the_v1_only_limit_reached_code() { as_v0[at + 15] = b'0'; assert!(decode_cultnet_message_from_slice(&as_v0, CultNetWireContract::CultNetSchemaV0).is_err()); } + +/// Vectors `admittedMessageId`: a write_admitted messageId needs a letter or number and no Default_Ignorable code +/// point, compared raw (never trimmed). The frame is the spec's `write_admitted_put` bytes with the id swapped in; a +/// case with `wireHex` (the lone surrogate, invalid UTF-8) supplies the whole frame. +#[test] +fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + let vectors: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = vectors["admittedMessageId"]["cases"].as_array().unwrap(); + assert_eq!(vectors["counts"]["admittedMessageIdCases"].as_u64().unwrap() as usize, cases.len()); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let mut failures = Vec::new(); + for case in cases { + let name = case["name"].as_str().unwrap(); + let id: String = case["codePoints"] + .as_array() + .unwrap() + .iter() + .filter_map(|cp| char::from_u32(u32::from_str_radix(cp.as_str().unwrap(), 16).unwrap())) + .collect::() + .repeat(case["repeat"].as_u64().unwrap_or(1) as usize); + let frame = match case["wireHex"].as_str() { + Some(hex) => unhex(hex), + None => { + let len = id.len(); + let header: Vec = if len < 32 { + vec![0xA0 | len as u8] + } else if len < 256 { + vec![0xD9, len as u8] + } else { + vec![0xDA, (len >> 8) as u8, len as u8] + }; + [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat() + } + }; + let decoded = decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0); + match (case["expect"].as_str().unwrap(), decoded) { + ("refuse", Ok(_)) => failures.push(format!("{name}: expected refuse")), + ("accept", Err(error)) => failures.push(format!("{name}: expected accept, got {error}")), + ("accept", Ok(CultNetMessage::WriteAdmitted { message_id, .. })) if message_id != id => { + failures.push(format!("{name}: id not echoed raw")) + } + _ => {} + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 6acb79cf..fbc8b6f0 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -280,3 +280,24 @@ test("a blank messageId, record key or tag is exactly Unicode White_Space, not t parseCultNetMessage(put({ tags: [named] })); } }); + +// `admittedMessageId` vectors: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, +// compared raw (never trimmed). JavaScript strings hold a lone surrogate, so the D800 vector is built from its code point. +test("write_admitted messageId: every admittedMessageId vector is refused or accepted raw", () => { + const vectorsPath = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); + assert.equal(vectors.counts.admittedMessageIdCases, vectors.admittedMessageId.cases.length); + const failures: string[] = []; + for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; expect: "refuse" | "accept" }[]) { + const id = vector.codePoints.map((cp) => String.fromCodePoint(parseInt(cp, 16))).join("").repeat(vector.repeat ?? 1); + let decodedId: string | undefined; + try { + decodedId = (parseCultNetMessage(admitted(id)) as CultNetWriteAdmittedMessage).messageId; + } catch { + decodedId = undefined; + } + if ((decodedId === undefined) !== (vector.expect === "refuse")) failures.push(`${vector.name}: expected ${vector.expect}`); + else if (decodedId !== undefined && decodedId !== id) failures.push(`${vector.name}: id not echoed raw`); + } + assert.deepEqual(failures, []); +}); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs new file mode 100644 index 00000000..4080f0d3 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // contracts/cultnet/interop/write-receipt-vectors.json `admittedMessageId`: a write_admitted messageId needs a letter or + // number and no Default_Ignorable code point, compared raw (never trimmed). The frame is built with this runtime's own + // encoder; the case with `wireHex` (a lone surrogate is invalid UTF-8) supplies the whole frame. + public sealed class CultNetWriteAdmittedMessageIdTests + { + private static JsonElement Vectors() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var path = Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + return JsonDocument.Parse(File.ReadAllText(path)).RootElement.Clone(); + } + + public static IEnumerable Cases() => + Vectors().GetProperty("admittedMessageId").GetProperty("cases").EnumerateArray().Select(c => c.GetProperty("name").GetString()!); + + [Test] + public void TheFileDeclaresHowManyCasesThisRunnerMustRun() => + Assert.That(Cases().Count(), Is.EqualTo(Vectors().GetProperty("counts").GetProperty("admittedMessageIdCases").GetInt32())); + + [TestCaseSource(nameof(Cases))] + public void AWriteAdmittedMessageIdIsRefusedOrAcceptedRaw(string name) + { + var vector = Vectors().GetProperty("admittedMessageId").GetProperty("cases").EnumerateArray() + .Single(c => c.GetProperty("name").GetString() == name); + var id = string.Concat(Enumerable.Repeat( + string.Concat(vector.GetProperty("codePoints").EnumerateArray() + .Select(cp => char.ConvertFromUtf32(Convert.ToInt32(cp.GetString(), 16)))), + vector.TryGetProperty("repeat", out var repeat) ? repeat.GetInt32() : 1)); + var bytes = vector.TryGetProperty("wireHex", out var wire) + ? Convert.FromHexString(wire.GetString()!) + : CultNetSchemaMessageSerialization.Serialize(new CultNetWriteAdmittedMessage + { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" }); + + CultNetWriteAdmittedMessage? decoded = null; + try { decoded = (CultNetWriteAdmittedMessage)CultNetSchemaMessageSerialization.Deserialize(bytes); } + catch (Exception) { /* a refusal */ } + + if (vector.GetProperty("expect").GetString() == "refuse") + Assert.That(decoded, Is.Null, vector.GetProperty("why").GetString()); + else + { + Assert.That(decoded, Is.Not.Null, vector.GetProperty("why").GetString()); + Assert.That(decoded!.MessageId, Is.EqualTo(id)); + } + } + } +} From 70f8135d2c713d33d203ecf0f6a7d41b2b59c674 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:41:11 +0000 Subject: [PATCH 095/117] vector: write_admitted messageId accepts a NUL between letters (rule: letter present, no Default_Ignorable) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt-vectors.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index a37d9660..7fa2e2fa 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -412,6 +412,16 @@ "expect": "accept", "why": "a combining mark after a letter is allowed" }, + { + "name": "accept_letter_nul_letter", + "codePoints": [ + "0061", + "0000", + "0062" + ], + "expect": "accept", + "why": "U+0000 is not Default_Ignorable and a letter is present, so the rule accepts; raw compare keeps the NUL" + }, { "name": "accept_256_ascii_letters", "codePoints": [ @@ -1457,7 +1467,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 32 + "admittedMessageIdCases": 33 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", From 4c1537885dd61f13628651ecf4633bc97fc10095 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:42:01 +0000 Subject: [PATCH 096/117] cultnet-ts: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare, no trim. The platform's regex property tables carry both properties; no table embedded. D800 vector now driven from wireHex. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 13 +++++++++++++ packages/cultnet-ts/test/write-receipt.test.ts | 8 +++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 3cc87dcd..a05a26df 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -950,6 +950,10 @@ export function parseCultNetMessage(input: unknown, wireContract: CultNetWireCon return input as CultNetDatabaseChangeRawMessage; } + if (schemaVersion === "cultnet.write_admitted.v0" && !isValidAdmittedMessageId((input as { messageId?: unknown }).messageId)) { + throw new Error("Validation failed for cultnet.write_admitted.v0: /messageId: must contain a letter or number and no Default_Ignorable code point"); + } + if (!validator(input)) { const keys = Object.keys(input as Record).join(","); throw new Error(`${renderValidationErrors(schemaVersion, validator)}; keys=${keys}`); @@ -1460,6 +1464,15 @@ function validateRawDocumentRecord( requireOptionalStringArray(candidate.tags, `${instancePath}/tags`); } +/** + * A write_admitted messageId is valid iff it holds a Unicode letter or number and no Default_Ignorable_Code_Point, compared raw + * (write-receipt.md; the id is echoed byte for byte, so nothing is trimmed). The platform's regex tables carry both properties, + * so this runtime embeds no table; the Kotlin, Python, C# and Rust decoders embed DerivedCoreProperties ranges. + */ +function isValidAdmittedMessageId(value: unknown): boolean { + return typeof value === "string" && /[\p{L}\p{N}]/u.test(value) && !/\p{Default_Ignorable_Code_Point}/u.test(value); +} + /** Blank is empty or only Unicode White_Space (write-receipt.md, rule 5), not `trim()`: JS trims U+FEFF and keeps U+0085. */ function isBlank(value: string): boolean { return /^\p{White_Space}*$/u.test(value); diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index fbc8b6f0..c447367a 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { decode } from "@msgpack/msgpack"; import { CULTNET_ERROR_CODES, CULTNET_ERROR_V1_CODES, @@ -282,17 +283,18 @@ test("a blank messageId, record key or tag is exactly Unicode White_Space, not t }); // `admittedMessageId` vectors: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, -// compared raw (never trimmed). JavaScript strings hold a lone surrogate, so the D800 vector is built from its code point. +// compared raw (never trimmed). The D800 vector is the frame's bytes (wireHex), decoded by the msgpack library like the others. test("write_admitted messageId: every admittedMessageId vector is refused or accepted raw", () => { const vectorsPath = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); assert.equal(vectors.counts.admittedMessageIdCases, vectors.admittedMessageId.cases.length); const failures: string[] = []; - for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; expect: "refuse" | "accept" }[]) { + for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; wireHex?: string; expect: "refuse" | "accept" }[]) { const id = vector.codePoints.map((cp) => String.fromCodePoint(parseInt(cp, 16))).join("").repeat(vector.repeat ?? 1); let decodedId: string | undefined; try { - decodedId = (parseCultNetMessage(admitted(id)) as CultNetWriteAdmittedMessage).messageId; + const frame = vector.wireHex ? decode(Uint8Array.from(Buffer.from(vector.wireHex, "hex"))) : admitted(id); + decodedId = (parseCultNetMessage(frame) as CultNetWriteAdmittedMessage).messageId; } catch { decodedId = undefined; } From bf55c10199d98bf9180b2ca3a041a36409c44c8a Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:42:44 +0000 Subject: [PATCH 097/117] vector: write_admitted messageId refuses U+FEFF before a letter (kills a trim-before-check decoder) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt-vectors.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 7fa2e2fa..2958990b 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -351,6 +351,15 @@ "expect": "refuse", "why": "U+2060 is Default_Ignorable" }, + { + "name": "refuse_bom_before_letter", + "codePoints": [ + "FEFF", + "0061" + ], + "expect": "refuse", + "why": "U+FEFF is Default_Ignorable; JavaScript trim() would strip it and accept the rest, but the compare is raw" + }, { "name": "accept_abc", "codePoints": [ @@ -1467,7 +1476,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 33 + "admittedMessageIdCases": 34 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", From 688562c3164d2b68dfda4037fad25c6371092c74 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:43:26 +0000 Subject: [PATCH 098/117] cultnet-py: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare. unicodedata for L/N, an embedded Default_Ignorable range table (Unicode 16.0). Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet-py/src/cultnet_py/messages.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index f2d89296..8ece7f6e 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import unicodedata from dataclasses import dataclass from typing import Any @@ -69,6 +70,31 @@ def _is_name(value: Any) -> bool: return isinstance(value, str) and value != "" +# Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). This runtime has no +# platform table for the property, so it embeds it; the Kotlin, C# and Rust decoders embed the same ranges, TypeScript uses its +# regex engine's. Letters and numbers come from unicodedata (categories L* and N*); a code point the running Python's Unicode +# data does not assign (U+0378 in every version) is Cn, so it is refused either way. +_DEFAULT_IGNORABLE = ( + (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +) + + +def _is_admitted_message_id(value: Any) -> bool: + """write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never stripped).""" + if not isinstance(value, str): + return False + has_letter_or_number = False + for ch in value: + point = ord(ch) + if any(low <= point <= high for low, high in _DEFAULT_IGNORABLE): + return False + if unicodedata.category(ch)[0] in ("L", "N"): + has_letter_or_number = True + return has_letter_or_number + + def _validate_write_admitted_body(body: dict[str, Any]) -> None: """Refuse a cultnet.write_admitted.v0 body that is not exactly the five keys of its closed schema.""" expected = ("messageId", "operation", "schemaId", "recordKey") @@ -77,6 +103,8 @@ def _validate_write_admitted_body(body: dict[str, Any]) -> None: for key in ("messageId", "schemaId", "recordKey"): if not _is_name(body[key]): raise ValueError(f"cultnet.write_admitted.v0 {key} must be a non-empty string") + if not _is_admitted_message_id(body["messageId"]): + raise ValueError("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point") if body["operation"] not in ("put", "delete"): raise ValueError("cultnet.write_admitted.v0 operation must be put or delete") From d1a4ebdd6860defc13ebd7aa03fbf4ed5c2ff1fc Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:45:37 +0000 Subject: [PATCH 099/117] cultmesh-kotlin: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare. JDK Character categories for L/N, an embedded Default_Ignorable range table (Unicode 16.0). The MessagePack string reader now refuses invalid UTF-8 instead of substituting U+FFFD. Co-Authored-By: Claude Sonnet 5.5 --- .../kotlin/org/gamecult/cultmesh/CultMesh.kt | 5 +++- .../org/gamecult/cultmesh/WriteReceipt.kt | 27 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 4a06d7f5..2337448d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -19,6 +19,7 @@ import java.net.Socket import java.net.SocketTimeoutException import java.net.URI import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction import java.nio.charset.StandardCharsets import java.security.SecureRandom import java.security.MessageDigest @@ -6306,7 +6307,9 @@ class MessagePackReader(payload: ByteArray) { code == 0xdb -> input.readInt() else -> throw IOException("expected string") } - return String(readPayload(length), StandardCharsets.UTF_8) + // Strict: bytes that are not UTF-8 (a lone surrogate's ED A0 80) refuse the frame instead of becoming U+FFFD. + return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(readPayload(length))).toString() } private fun readBinaryAfterCode(code: Int): ByteArray { diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt index 4b38d942..523dfc2e 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -74,7 +74,32 @@ fun decodeWriteAdmitted(message: CultNetMessage): CultNetWriteAdmitted { closedKeys(message, setOf("messageId", "operation", "schemaId", "recordKey")) val operation = nonEmpty(message, "operation") if (operation != "put" && operation != "delete") throw IOException("operation must be put or delete") - return CultNetWriteAdmitted(nonEmpty(message, "messageId"), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) + return CultNetWriteAdmitted(admittedMessageId(message), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) +} + +// Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). The JDK has no such +// property, so this runtime embeds it; the Python, C# and Rust decoders embed the same ranges, TypeScript uses its regex +// engine's. Letters and numbers come from the JDK's Character categories (L*, and Nd, Nl, No); a code point the running JDK +// does not assign (U+0378 in every version) has no such category, so it is refused either way. +private val DEFAULT_IGNORABLE: List = listOf( + 0x00AD..0x00AD, 0x034F..0x034F, 0x061C..0x061C, 0x115F..0x1160, 0x17B4..0x17B5, 0x180B..0x180F, + 0x200B..0x200F, 0x202A..0x202E, 0x2060..0x206F, 0x3164..0x3164, 0xFE00..0xFE0F, 0xFEFF..0xFEFF, + 0xFFA0..0xFFA0, 0xFFF0..0xFFF8, 0x1BCA0..0x1BCA3, 0x1D173..0x1D17A, 0xE0000..0xE0FFF, +) + +/** write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never trimmed). */ +private fun admittedMessageId(message: CultNetMessage): String { + val id = nonEmpty(message, "messageId") + var hasLetterOrNumber = false + id.codePoints().forEach { cp -> + if (DEFAULT_IGNORABLE.any { cp in it }) throw IOException("messageId holds a Default_Ignorable code point") + when (Character.getType(cp).toByte()) { + Character.UPPERCASE_LETTER, Character.LOWERCASE_LETTER, Character.TITLECASE_LETTER, Character.MODIFIER_LETTER, + Character.OTHER_LETTER, Character.DECIMAL_DIGIT_NUMBER, Character.LETTER_NUMBER, Character.OTHER_NUMBER -> hasLetterOrNumber = true + } + } + if (!hasLetterOrNumber) throw IOException("messageId must contain a letter or number") + return id } /** Strict decode of error.v0 (no messageId, no limit_reached) or error.v1 (messageId required, string or nil, never empty). */ From d546ef9c7fb2dcdfc8ce1336a5ffd1277f333d66 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:49:50 +0000 Subject: [PATCH 100/117] GameCult.Networking: refuse a write_admitted messageId without a letter or number, with a Default_Ignorable code point, or with invalid UTF-8 Raw compare. .NET Unicode categories for L/N, an embedded Default_Ignorable range table (Unicode 16.0). Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 69 +++++++++++++++++++ .../CultNetWriteAdmittedMessageIdTests.cs | 2 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index cdcca912..9789e05e 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -1,4 +1,6 @@ using System; +using System.Buffers; +using System.Globalization; using MessagePack; namespace GameCult.Networking @@ -29,6 +31,7 @@ namespace GameCult.Networking public static T Deserialize(byte[] payload) where T : ICultNetSchemaMessage { if (payload == null) throw new ArgumentNullException(nameof(payload)); + if (typeof(T) == typeof(CultNetWriteAdmittedMessage)) CultNetAdmittedMessageId.Require(payload); return MessagePackSerializer.Deserialize(payload, Options); } @@ -98,4 +101,70 @@ namespace GameCult.Networking [Key("schemaVersion")] public string SchemaVersion { get; set; } = string.Empty; } } + + /// + /// The write_admitted messageId rule (write-receipt.md): a Unicode letter or number somewhere, no Default_Ignorable code + /// point anywhere, wire bytes that are valid UTF-8, compared raw (never trimmed). Letters and numbers come from the + /// runtime's Unicode categories (L*, Nd, Nl, No); a code point it does not assign (U+0378 in every version) has neither. + /// .NET has no Default_Ignorable property, so the ranges are embedded from DerivedCoreProperties.txt, Unicode 16.0 + /// (unchanged since 15.1); the Kotlin, Python and Rust decoders embed the same ranges, TypeScript uses its regex engine's. + /// + internal static class CultNetAdmittedMessageId + { + private static readonly (int Low, int High)[] DefaultIgnorable = + { + (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), + }; + + private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); + + /// Throws unless the frame's top-level messageId is a valid write_admitted id; other keys are left to the serializer. + public static void Require(byte[] payload) + { + var reader = new MessagePackReader(payload); + var entries = reader.ReadMapHeader(); + for (var index = 0; index < entries; index++) + { + var key = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; + if (key == "messageId" && reader.NextMessagePackType == MessagePackType.String) + { + string id; + try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } + catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } + if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); + return; + } + + if (key == null) reader.Skip(); + reader.Skip(); + } + } + + public static bool IsValid(string id) + { + var hasLetterOrNumber = false; + for (var at = 0; at < id.Length; at += char.IsSurrogatePair(id, at) ? 2 : 1) + { + var point = char.IsSurrogatePair(id, at) ? char.ConvertToUtf32(id, at) : id[at]; + if (Array.Exists(DefaultIgnorable, range => point >= range.Low && point <= range.High)) return false; + switch (CharUnicodeInfo.GetUnicodeCategory(point)) + { + case UnicodeCategory.UppercaseLetter: + case UnicodeCategory.LowercaseLetter: + case UnicodeCategory.TitlecaseLetter: + case UnicodeCategory.ModifierLetter: + case UnicodeCategory.OtherLetter: + case UnicodeCategory.DecimalDigitNumber: + case UnicodeCategory.LetterNumber: + case UnicodeCategory.OtherNumber: + hasLetterOrNumber = true; + break; + } + } + + return hasLetterOrNumber; + } + } } diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index 4080f0d3..f5cac528 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -35,7 +35,7 @@ namespace GameCult.Networking.Tests .Single(c => c.GetProperty("name").GetString() == name); var id = string.Concat(Enumerable.Repeat( string.Concat(vector.GetProperty("codePoints").EnumerateArray() - .Select(cp => char.ConvertFromUtf32(Convert.ToInt32(cp.GetString(), 16)))), + .Select(cp => Convert.ToInt32(cp.GetString(), 16)).Where(point => point < 0xD800 || point > 0xDFFF).Select(char.ConvertFromUtf32)), vector.TryGetProperty("repeat", out var repeat) ? repeat.GetInt32() : 1)); var bytes = vector.TryGetProperty("wireHex", out var wire) ? Convert.FromHexString(wire.GetString()!) From ba548d192df76bb533ec8bcdb119088693a1381e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:49:50 +0000 Subject: [PATCH 101/117] cultnet-rs: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare. std is_numeric for N; embedded letter (Unicode 15.1) and Default_Ignorable (16.0) range tables. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-rs/src/contracts.rs | 162 ++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 0ea2aa53..201632a2 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -680,7 +680,7 @@ fn validate_message(message: &CultNetMessage) -> Result<()> { } } CultNetMessage::WriteAdmitted { message_id, operation, schema_id, record_key } => { - require_non_empty(message_id, "messageId")?; + require_admitted_message_id(message_id)?; if operation != "put" && operation != "delete" { return Err(anyhow!("operation must be put or delete")); } @@ -934,6 +934,166 @@ fn require_non_empty(value: &str, field: &str) -> Result<()> { Ok(()) } +// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (\p{L}) or number (\p{N}) and no +// Default_Ignorable_Code_Point, compared raw (never trimmed). std has `char::is_numeric` (N) but no letter category +// (`is_alphabetic` is the Alphabetic property, which also takes combining marks), and no Default_Ignorable, so both tables +// are embedded: DerivedCoreProperties.txt ranges, Unicode 16.0 (unchanged since 15.1), and the general category L ranges of +// Unicode 15.1 (U+0378 is unassigned in every version, so it is refused either way). The Kotlin, Python and C# decoders embed +// the same Default_Ignorable ranges and take L/N from their platform; TypeScript uses its regex engine's tables. +// Invalid UTF-8 never reaches this check: the MessagePack str decode refuses it. +const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ + (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +]; + +const LETTERS: &[(u32, u32)] = &[ + (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB5, 0xB5), (0xBA, 0xBA), (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), + (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), + (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), + (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), + (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), (0x6E5, 0x6E6), (0x6EE, 0x6EF), (0x6FA, 0x6FC), (0x6FF, 0x6FF), + (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), (0x7CA, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), + (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), + (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), (0x950, 0x950), (0x958, 0x961), (0x971, 0x980), + (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), + (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), (0x9F0, 0x9F1), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), + (0xA13, 0xA28), (0xA2A, 0xA30), (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), + (0xA72, 0xA74), (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), + (0xABD, 0xABD), (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), + (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB71, 0xB71), + (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), (0xB9E, 0xB9F), + (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xC05, 0xC0C), (0xC0E, 0xC10), (0xC12, 0xC28), + (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), (0xC80, 0xC80), (0xC85, 0xC8C), + (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), + (0xCF1, 0xCF2), (0xD04, 0xD0C), (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), + (0xD5F, 0xD61), (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), + (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), (0xE8C, 0xEA3), + (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), (0xEDC, 0xEDF), + (0xF00, 0xF00), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), (0x1000, 0x102A), (0x103F, 0x103F), + (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), (0x106E, 0x1070), (0x1075, 0x1081), + (0x108E, 0x108E), (0x10A0, 0x10C5), (0x10C7, 0x10C7), (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), + (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), + (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), + (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), + (0x1401, 0x166C), (0x166F, 0x167F), (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16F1, 0x16F8), (0x1700, 0x1711), + (0x171F, 0x1731), (0x1740, 0x1751), (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), + (0x17DC, 0x17DC), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), (0x18B0, 0x18F5), + (0x1900, 0x191E), (0x1950, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), (0x1A00, 0x1A16), + (0x1A20, 0x1A54), (0x1AA7, 0x1AA7), (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B83, 0x1BA0), (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), (0x1C00, 0x1C23), (0x1C4D, 0x1C4F), (0x1C5A, 0x1C7D), (0x1C80, 0x1C88), (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), (0x1F18, 0x1F1D), (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), (0x2071, 0x2071), (0x207F, 0x207F), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), + (0x210A, 0x2113), (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), + (0x212A, 0x212D), (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2183, 0x2184), + (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), + (0x3005, 0x3006), (0x3031, 0x3035), (0x303B, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), + (0x30FC, 0x30FF), (0x3105, 0x312F), (0x3131, 0x318E), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3400, 0x4DBF), + (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), (0xA500, 0xA60C), (0xA610, 0xA61F), (0xA62A, 0xA62B), (0xA640, 0xA66E), + (0xA67F, 0xA69D), (0xA6A0, 0xA6E5), (0xA717, 0xA71F), (0xA722, 0xA788), (0xA78B, 0xA7CA), (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), (0xA7D5, 0xA7D9), (0xA7F2, 0xA801), (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), + (0xA840, 0xA873), (0xA882, 0xA8B3), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA90A, 0xA925), + (0xA930, 0xA946), (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9CF), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), (0xAA00, 0xAA28), (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), + (0xAB20, 0xAB26), (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), + (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), (0x10028, 0x1003A), + (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), (0x10280, 0x1029C), + (0x102A0, 0x102D0), (0x10300, 0x1031F), (0x1032D, 0x10340), (0x10342, 0x10349), (0x10350, 0x10375), + (0x10380, 0x1039D), (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x10400, 0x1049D), (0x104B0, 0x104D3), + (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), (0x1057C, 0x1058A), + (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), (0x105B3, 0x105B9), + (0x105BB, 0x105BC), (0x10600, 0x10736), (0x10740, 0x10755), (0x10760, 0x10767), (0x10780, 0x10785), + (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), (0x10808, 0x10808), (0x1080A, 0x10835), + (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), (0x10860, 0x10876), (0x10880, 0x1089E), + (0x108E0, 0x108F2), (0x108F4, 0x108F5), (0x10900, 0x10915), (0x10920, 0x10939), (0x10980, 0x109B7), + (0x109BE, 0x109BF), (0x10A00, 0x10A00), (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), (0x10A80, 0x10A9C), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10B00, 0x10B35), + (0x10B40, 0x10B55), (0x10B60, 0x10B72), (0x10B80, 0x10B91), (0x10C00, 0x10C48), (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), (0x10D00, 0x10D23), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), (0x10F30, 0x10F45), (0x10F70, 0x10F81), (0x10FB0, 0x10FC4), (0x10FE0, 0x10FF6), + (0x11003, 0x11037), (0x11071, 0x11072), (0x11075, 0x11075), (0x11083, 0x110AF), (0x110D0, 0x110E8), + (0x11103, 0x11126), (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), + (0x11183, 0x111B2), (0x111C1, 0x111C4), (0x111DA, 0x111DA), (0x111DC, 0x111DC), (0x11200, 0x11211), + (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), + (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x11305, 0x1130C), (0x1130F, 0x11310), + (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), (0x1133D, 0x1133D), + (0x11350, 0x11350), (0x1135D, 0x11361), (0x11400, 0x11434), (0x11447, 0x1144A), (0x1145F, 0x11461), + (0x11480, 0x114AF), (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x11580, 0x115AE), (0x115D8, 0x115DB), + (0x11600, 0x1162F), (0x11644, 0x11644), (0x11680, 0x116AA), (0x116B8, 0x116B8), (0x11700, 0x1171A), + (0x11740, 0x11746), (0x11800, 0x1182B), (0x118A0, 0x118DF), (0x118FF, 0x11906), (0x11909, 0x11909), + (0x1190C, 0x11913), (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), + (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), (0x11C40, 0x11C40), (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), (0x11D08, 0x11D09), (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D60, 0x11D65), + (0x11D67, 0x11D68), (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), + (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11FB0, 0x11FB0), (0x12000, 0x12399), (0x12480, 0x12543), + (0x12F90, 0x12FF0), (0x13000, 0x1342F), (0x13441, 0x13446), (0x14400, 0x14646), (0x16800, 0x16A38), + (0x16A40, 0x16A5E), (0x16A70, 0x16ABE), (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), + (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16E40, 0x16E7F), (0x16F00, 0x16F4A), (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), (0x17000, 0x187F7), (0x18800, 0x18CD5), + (0x18D00, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), + (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), (0x1BC90, 0x1BC99), (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), (0x1D540, 0x1D544), + (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), (0x1E137, 0x1E13D), (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), (0x1E4D0, 0x1E4EB), (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x20000, 0x2A6DF), (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x323AF), +]; + +fn in_ranges(ranges: &[(u32, u32)], point: u32) -> bool { + ranges.binary_search_by(|&(low, high)| { + if point < low { + std::cmp::Ordering::Greater + } else if point > high { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} + +fn require_admitted_message_id(value: &str) -> Result<()> { + let mut has_letter_or_number = false; + for ch in value.chars() { + let point = ch as u32; + if in_ranges(DEFAULT_IGNORABLE, point) { + return Err(anyhow!("CultNet field messageId must not hold a Default_Ignorable code point")); + } + has_letter_or_number |= ch.is_numeric() || in_ranges(LETTERS, point); + } + if !has_letter_or_number { + return Err(anyhow!("CultNet field messageId must contain a letter or number")); + } + Ok(()) +} + fn require_exact(value: &str, expected: &str, field: &str) -> Result<()> { if value != expected { return Err(anyhow!("{field} must be {expected}")); From cc931932fc0bf32adaecac74345db84edf4314e1 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:00:54 +0000 Subject: [PATCH 102/117] tests first: Unicode 16.0 pin vectors (failing) Failing before the fix: Rust 1 (16100), Python 2 (16100, 10D40), Kotlin 4 (16100, 10D40, 2EBF0, 2EE5D). TypeScript and C# already pass: their platform data is 16.0 or later. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 2958990b..c7f47738 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -150,7 +150,7 @@ ] }, "admittedMessageId": { - "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Scope: write_admitted only; error.v1 is untouched.", + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Letters, numbers and Default_Ignorable are those of Unicode 16.0 in every runtime, from embedded tables (no platform category lookup), so a runtime's own Unicode version cannot change a result. Scope: write_admitted only; error.v1 is untouched.", "cases": [ { "name": "refuse_braille_blank_2800", @@ -360,6 +360,54 @@ "expect": "refuse", "why": "U+FEFF is Default_Ignorable; JavaScript trim() would strip it and accept the rest, but the compare is raw" }, + { + "name": "accept_unicode16_gurung_khema_letter_16100", + "codePoints": [ + "16100" + ], + "expect": "accept", + "why": "U+16100 GURUNG KHEMA LETTER A is Lo and new in Unicode 16.0 (unassigned in 15.1); the rule is pinned to 16.0 in every runtime" + }, + { + "name": "accept_unicode16_garay_digit_zero_10D40", + "codePoints": [ + "10D40" + ], + "expect": "accept", + "why": "U+10D40 GARAY DIGIT ZERO is Nd and new in Unicode 16.0" + }, + { + "name": "accept_unicode15_1_cjk_ext_i_2EBF0", + "codePoints": [ + "2EBF0" + ], + "expect": "accept", + "why": "U+2EBF0, first of CJK Unified Ideographs Extension I, is Lo and new in Unicode 15.1" + }, + { + "name": "accept_last_cjk_ext_i_2EE5D", + "codePoints": [ + "2EE5D" + ], + "expect": "accept", + "why": "U+2EE5D is the last assigned code point of Extension I (Lo)" + }, + { + "name": "refuse_after_cjk_ext_i_2EE5E", + "codePoints": [ + "2EE5E" + ], + "expect": "refuse", + "why": "U+2EE5E is unassigned in Unicode 16.0, so neither a letter nor a number" + }, + { + "name": "refuse_unassigned_in_16_0_10D90", + "codePoints": [ + "10D90" + ], + "expect": "refuse", + "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0" + }, { "name": "accept_abc", "codePoints": [ @@ -1476,7 +1524,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 34 + "admittedMessageIdCases": 40 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", From 3d37fda49638c4171ab654b1a72388aaee175449 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 103/117] write_admitted messageId: pin Unicode 16.0, one generator for the embedded tables scripts/gen-unicode-messageid-tables.py emits each runtime's Letter-or-Number and Default_Ignorable tables from the two UCD 16.0 files; --check fails on a stale or hand-edited table. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- scripts/gen-unicode-messageid-tables.py | 111 +++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 scripts/gen-unicode-messageid-tables.py diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 150c10f8..10f42a6a 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. diff --git a/scripts/gen-unicode-messageid-tables.py b/scripts/gen-unicode-messageid-tables.py new file mode 100644 index 00000000..96d1a205 --- /dev/null +++ b/scripts/gen-unicode-messageid-tables.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Generate the Unicode 16.0 tables of the write_admitted messageId rule for the five CultNet runtimes. + +The rule (contracts/cultnet/interop/write-receipt.md): a messageId is valid iff it holds a letter (Lu Ll Lt Lm Lo) or a number +(Nd Nl No) and no Default_Ignorable_Code_Point, all as of Unicode 16.0. The tables are embedded in each runtime so that no +platform's own Unicode version can change a result. Bumping the version is one coordinated change: run this script on the new +UCD files, commit the five regenerated files and the vectors that pin the new version, together. + + gen-unicode-messageid-tables.py DerivedGeneralCategory.txt DerivedCoreProperties.txt [--check] + +The inputs are https://www.unicode.org/Public/16.0.0/ucd/extracted/DerivedGeneralCategory.txt and +https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --check nothing is written; the exit status is 1 if +any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only. +""" +import hashlib +import re +import sys +from pathlib import Path + +VERSION = "16.0.0" +ROOT = Path(__file__).resolve().parent.parent +LETTER_OR_NUMBER_CATEGORIES = {"Lu", "Ll", "Lt", "Lm", "Lo", "Nd", "Nl", "No"} + +TARGETS = { + "ts": ROOT / "packages/cultnet-ts/src/unicode-messageid-tables.ts", + "py": ROOT / "packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py", + "kt": ROOT / "packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt", + "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs", + "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs", +} + + +def parse(path, wanted): + """Ranges (low, high) of the lines `LOW[..HIGH] ; VALUE`, VALUE accepted by [wanted], merged and sorted.""" + points = [] + for line in Path(path).read_text(encoding="utf-8").splitlines(): + m = re.match(r"([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*(\S+)", line) + if m and wanted(m.group(3)): + points.append((int(m.group(1), 16), int(m.group(2) or m.group(1), 16))) + points.sort() + merged = [] + for low, high in points: + if merged and low <= merged[-1][1] + 1: + merged[-1] = (merged[-1][0], max(merged[-1][1], high)) + else: + merged.append((low, high)) + return merged + + +def flat(ranges): + return [value for pair in ranges for value in pair] + + +def wrapped(items, indent, render, width=118): + lines, current = [], indent + for item in items: + text = render(item) + ", " + if len(current) + len(text) > width: + lines.append(current.rstrip()) + current = indent + current += text + lines.append(current.rstrip()) + return "\n".join(lines) + + +def emit(target, letters, ignorable, header): + num = lambda v: f"0x{v:X}" + if target == "ts": + body = lambda name, r: f"export const {name}: readonly number[] = [\n{wrapped(flat(r), ' ', num)}\n];\n" + return f"// {header}\n\n// [low, high] pairs, sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "py": + body = lambda name, r: f"{name}: tuple[int, ...] = (\n{wrapped(flat(r), ' ', num)}\n)\n" + return f"# {header}\n\n# (low, high) pairs, flattened and sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "kt": + body = lambda name, r: f"internal val {name}: IntArray = intArrayOf(\n{wrapped(flat(r), ' ', num)}\n)\n" + return f"// {header}\n\npackage org.gamecult.cultmesh\n\n// (low, high) pairs, flattened and sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "cs": + body = lambda name, r: f" public static readonly int[] {name} =\n {{\n{wrapped(flat(r), ' ', num)}\n }};\n" + return f"// {header}\n\nnamespace GameCult.Networking\n{{\n internal static class CultNetUnicodeMessageIdTables\n {{\n // (low, high) pairs, flattened and sorted.\n{body('LetterOrNumber', letters)}\n{body('DefaultIgnorable', ignorable)} }}\n}}\n" + body = lambda name, r: f"pub const {name}: &[(u32, u32)] = &[\n{wrapped(r, ' ', lambda p: f'({num(p[0])}, {num(p[1])})')}\n];\n" + return f"// {header}\n\n// (low, high) pairs, sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + + +def main(): + args = [a for a in sys.argv[1:] if a != "--check"] + check = "--check" in sys.argv[1:] + if len(args) != 2: + sys.exit(__doc__) + gc, dcp = args + for path, marker in ((gc, f"DerivedGeneralCategory-{VERSION}"), (dcp, f"DerivedCoreProperties-{VERSION}")): + if marker not in Path(path).read_text(encoding="utf-8").splitlines()[0]: + sys.exit(f"{path} is not {marker}.txt") + letters = parse(gc, lambda v: v in LETTER_OR_NUMBER_CATEGORIES) + ignorable = parse(dcp, lambda v: v == "Default_Ignorable_Code_Point") + digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16] + header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit" + stale = [] + for target, path in TARGETS.items(): + text = emit(target, letters, ignorable, header) + if check: + if not path.exists() or path.read_text(encoding="utf-8") != text: + stale.append(str(path.relative_to(ROOT))) + else: + path.write_text(text, encoding="utf-8") + if stale: + sys.exit("stale or hand-edited: " + ", ".join(stale)) + print(f"{'checked' if check else 'wrote'} {len(TARGETS)} tables: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") + + +if __name__ == "__main__": + main() From de25b50abdb4ac90b0512d7436aa59605ad9c678 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 104/117] cultnet-ts: write_admitted messageId from embedded Unicode 16.0 tables, not the regex engine's Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 38 +++++- .../src/unicode-messageid-tables.ts | 126 ++++++++++++++++++ 2 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 packages/cultnet-ts/src/unicode-messageid-tables.ts diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index a05a26df..09d5d823 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -10,6 +10,7 @@ import verifySchema from "../contracts/cultnet/cultnet.verify.schema.json"; import loginSuccessSchema from "../contracts/cultnet/cultnet.login-success.schema.json"; import errorSchema from "../contracts/cultnet/cultnet.error.schema.json"; import errorV1Schema from "../contracts/cultnet/interop/cultnet.error.v1.schema.json"; +import { DEFAULT_IGNORABLE, LETTER_OR_NUMBER } from "./unicode-messageid-tables"; import writeAdmittedSchema from "../contracts/cultnet/interop/cultnet.write-admitted.schema.json"; import sampleChangeNameSchema from "../contracts/cultnet/cultnet.sample-change-name.schema.json"; import sampleChatSchema from "../contracts/cultnet/cultnet.sample-chat.schema.json"; @@ -1465,12 +1466,41 @@ function validateRawDocumentRecord( } /** - * A write_admitted messageId is valid iff it holds a Unicode letter or number and no Default_Ignorable_Code_Point, compared raw - * (write-receipt.md; the id is echoed byte for byte, so nothing is trimmed). The platform's regex tables carry both properties, - * so this runtime embeds no table; the Kotlin, Python, C# and Rust decoders embed DerivedCoreProperties ranges. + * A write_admitted messageId is valid iff it holds a Unicode letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) and no + * Default_Ignorable_Code_Point, as of Unicode 16.0, compared raw (write-receipt.md; the id is echoed byte for byte, so nothing + * is trimmed). The tables are embedded (unicode-messageid-tables.ts, generated by scripts/gen-unicode-messageid-tables.py), not + * the regex engine's, so the running Node's Unicode version changes no result; the other four runtimes embed the same tables. */ function isValidAdmittedMessageId(value: unknown): boolean { - return typeof value === "string" && /[\p{L}\p{N}]/u.test(value) && !/\p{Default_Ignorable_Code_Point}/u.test(value); + if (typeof value !== "string") { + return false; + } + let hasLetterOrNumber = false; + for (const ch of value) { + const point = ch.codePointAt(0) as number; + if (inRanges(DEFAULT_IGNORABLE, point)) { + return false; + } + hasLetterOrNumber = hasLetterOrNumber || inRanges(LETTER_OR_NUMBER, point); + } + return hasLetterOrNumber; +} + +/** Whether [point] lies in the flattened, sorted [low, high] pairs. */ +function inRanges(ranges: readonly number[], point: number): boolean { + let low = 0; + let high = ranges.length / 2 - 1; + while (low <= high) { + const mid = (low + high) >> 1; + if (point < ranges[2 * mid]) { + high = mid - 1; + } else if (point > ranges[2 * mid + 1]) { + low = mid + 1; + } else { + return true; + } + } + return false; } /** Blank is empty or only Unicode White_Space (write-receipt.md, rule 5), not `trim()`: JS trims U+FEFF and keeps U+0085. */ diff --git a/packages/cultnet-ts/src/unicode-messageid-tables.ts b/packages/cultnet-ts/src/unicode-messageid-tables.ts new file mode 100644 index 00000000..3374a011 --- /dev/null +++ b/packages/cultnet-ts/src/unicode-messageid-tables.ts @@ -0,0 +1,126 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +// [low, high] pairs, sorted. +export const LETTER_OR_NUMBER: readonly number[] = [ + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, 0x103F, + 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, + 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, 0x124A, + 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, 0x12B2, + 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, 0x1318, + 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, 0x1681, + 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, 0x176E, + 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, 0x1820, + 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, 0x1970, + 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, 0x1A90, + 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1C00, + 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, 0x1CEE, + 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, + 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, 0x1FB6, + 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, 0x1FF2, + 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, 0x2107, + 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, + 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, 0x24EA, + 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, 0x2D27, + 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, 0x2DB0, + 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, 0x3005, + 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, 0x30FC, + 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, 0x3248, + 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, 0xA500, + 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, 0xA78B, + 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, + 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, 0xA8FD, + 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, 0xA9E6, + 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7E, + 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, 0xAAE0, + 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, 0xAB30, + 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0xF900, + 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, 0xFB38, + 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, 0xFD92, + 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, 0xFF66, + 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, 0x10026, + 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, 0x10133, + 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, 0x10323, + 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, 0x103D5, + 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, 0x10563, + 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, 0x105B1, + 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, + 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, + 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, 0x109CF, + 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, 0x10A7E, + 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, 0x10B55, + 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, + 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, 0x10EA9, + 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, 0x10F81, + 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, 0x11075, + 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, 0x11144, + 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, 0x111DA, + 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, 0x11286, + 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, 0x112F9, + 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, 0x11339, + 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, 0x1138E, + 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, 0x1144A, + 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, 0x114D9, + 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, 0x116AA, + 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, 0x11746, + 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, 0x11916, + 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D0, + 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, 0x11A50, + 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, + 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, 0x11D09, + 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, 0x11D89, + 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, 0x11F33, + 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, 0x12543, + 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, 0x1611D, + 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, 0x16AC9, + 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, + 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, 0x16F50, + 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, 0x18D08, + 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, 0x1B152, + 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, 0x1BC88, + 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, 0x1D4B9, + 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, 0x1D51C, + 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, 0x1D6A5, + 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, 0x1D74E, + 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, 0x1D7FF, + 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, 0x1E149, + 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, 0x1E4F9, + 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, 0x1E7FE, + 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, 0x1ECAB, + 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, + 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, 0x1EE39, + 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, 0x1EE4F, + 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, 0x1EE5D, + 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, 0x1EE77, + 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, 0x1EEA9, + 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, 0x2B81D, + 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, 0x323AF, +]; + +export const DEFAULT_IGNORABLE: readonly number[] = [ + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +]; From 87f53498ed1db7709ade57241bc9b47bf05619e0 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 105/117] cultnet-py: write_admitted messageId from embedded Unicode 16.0 tables, not unicodedata Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet-py/src/cultnet_py/messages.py | 34 +++-- .../cultnet_py/unicode_messageid_tables.py | 127 ++++++++++++++++++ 2 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index 8ece7f6e..d5e26b7a 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -1,12 +1,13 @@ from __future__ import annotations import hashlib -import unicodedata from dataclasses import dataclass from typing import Any import msgpack +from .unicode_messageid_tables import DEFAULT_IGNORABLE, LETTER_OR_NUMBER + @dataclass(frozen=True) class CultNetMessage: @@ -70,28 +71,33 @@ def _is_name(value: Any) -> bool: return isinstance(value, str) and value != "" -# Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). This runtime has no -# platform table for the property, so it embeds it; the Kotlin, C# and Rust decoders embed the same ranges, TypeScript uses its -# regex engine's. Letters and numbers come from unicodedata (categories L* and N*); a code point the running Python's Unicode -# data does not assign (U+0378 in every version) is Cn, so it is refused either way. -_DEFAULT_IGNORABLE = ( - (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), - (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), - (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), -) +# write-receipt.md: a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no Default_Ignorable code point anywhere, as of +# Unicode 16.0, compared raw (never stripped). The tables are embedded (unicode_messageid_tables.py, generated by +# scripts/gen-unicode-messageid-tables.py), not unicodedata's, so the running Python's Unicode version changes no result; +# the other four runtimes embed the same generated tables. +def _in_ranges(ranges: tuple[int, ...], point: int) -> bool: + """Whether [point] lies in the flattened, sorted (low, high) pairs.""" + low, high = 0, len(ranges) // 2 - 1 + while low <= high: + mid = (low + high) // 2 + if point < ranges[2 * mid]: + high = mid - 1 + elif point > ranges[2 * mid + 1]: + low = mid + 1 + else: + return True + return False def _is_admitted_message_id(value: Any) -> bool: - """write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never stripped).""" if not isinstance(value, str): return False has_letter_or_number = False for ch in value: point = ord(ch) - if any(low <= point <= high for low, high in _DEFAULT_IGNORABLE): + if _in_ranges(DEFAULT_IGNORABLE, point): return False - if unicodedata.category(ch)[0] in ("L", "N"): - has_letter_or_number = True + has_letter_or_number = has_letter_or_number or _in_ranges(LETTER_OR_NUMBER, point) return has_letter_or_number diff --git a/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py b/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py new file mode 100644 index 00000000..163e3c08 --- /dev/null +++ b/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py @@ -0,0 +1,127 @@ +# generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +# (low, high) pairs, flattened and sorted. +LETTER_OR_NUMBER: tuple[int, ...] = ( + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, + 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, + 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, + 0x124A, 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, + 0x12B2, 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, + 0x1318, 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, + 0x1681, 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, + 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, + 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, + 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, + 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, + 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, + 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, + 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, + 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, + 0x2107, 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, + 0x212A, 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, + 0x24EA, 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, + 0x2D27, 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, + 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, + 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, + 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, + 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, + 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, + 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, + 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, + 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, + 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, + 0xAAE0, 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB30, 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, + 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, + 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, + 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, + 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, + 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, + 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, + 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, + 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, + 0x10767, 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, + 0x10835, 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, + 0x108AF, 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, + 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, + 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, + 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, + 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, + 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, + 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, + 0x11144, 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, + 0x111DA, 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, + 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, + 0x112F9, 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, + 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, + 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, + 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, + 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, + 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, + 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, + 0x119D0, 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, + 0x11A50, 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, + 0x11C08, 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, + 0x11D09, 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, + 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, + 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, + 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, + 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, + 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, + 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, + 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, + 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, + 0x1BC88, 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, + 0x1D454, 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, + 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, + 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, + 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, + 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, + 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, + 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, + 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, + 0x1ECAB, 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, + 0x1EE1F, 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, + 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, + 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, + 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, + 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, + 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, + 0x323AF, +) + +DEFAULT_IGNORABLE: tuple[int, ...] = ( + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +) From c4f6e768395e2467346320c7ecff270791265ce5 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 106/117] cultmesh-kotlin: write_admitted messageId from embedded Unicode 16.0 tables, not the JDK's Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh/UnicodeMessageIdTables.kt | 129 ++++++++++++++++++ .../org/gamecult/cultmesh/WriteReceipt.kt | 34 +++-- 2 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt new file mode 100644 index 00000000..7bc4bdc8 --- /dev/null +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt @@ -0,0 +1,129 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +package org.gamecult.cultmesh + +// (low, high) pairs, flattened and sorted. +internal val LETTER_OR_NUMBER: IntArray = intArrayOf( + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, + 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, + 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, + 0x124A, 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, + 0x12B2, 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, + 0x1318, 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, + 0x1681, 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, + 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, + 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, + 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, + 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, + 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, + 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, + 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, + 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, + 0x2107, 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, + 0x212A, 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, + 0x24EA, 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, + 0x2D27, 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, + 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, + 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, + 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, + 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, + 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, + 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, + 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, + 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, + 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, + 0xAAE0, 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB30, 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, + 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, + 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, + 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, + 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, + 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, + 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, + 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, + 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, + 0x10767, 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, + 0x10835, 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, + 0x108AF, 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, + 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, + 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, + 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, + 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, + 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, + 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, + 0x11144, 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, + 0x111DA, 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, + 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, + 0x112F9, 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, + 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, + 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, + 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, + 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, + 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, + 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, + 0x119D0, 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, + 0x11A50, 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, + 0x11C08, 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, + 0x11D09, 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, + 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, + 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, + 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, + 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, + 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, + 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, + 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, + 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, + 0x1BC88, 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, + 0x1D454, 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, + 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, + 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, + 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, + 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, + 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, + 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, + 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, + 0x1ECAB, 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, + 0x1EE1F, 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, + 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, + 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, + 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, + 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, + 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, + 0x323AF, +) + +internal val DEFAULT_IGNORABLE: IntArray = intArrayOf( + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt index 523dfc2e..9cf9c402 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -77,26 +77,30 @@ fun decodeWriteAdmitted(message: CultNetMessage): CultNetWriteAdmitted { return CultNetWriteAdmitted(admittedMessageId(message), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) } -// Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). The JDK has no such -// property, so this runtime embeds it; the Python, C# and Rust decoders embed the same ranges, TypeScript uses its regex -// engine's. Letters and numbers come from the JDK's Character categories (L*, and Nd, Nl, No); a code point the running JDK -// does not assign (U+0378 in every version) has no such category, so it is refused either way. -private val DEFAULT_IGNORABLE: List = listOf( - 0x00AD..0x00AD, 0x034F..0x034F, 0x061C..0x061C, 0x115F..0x1160, 0x17B4..0x17B5, 0x180B..0x180F, - 0x200B..0x200F, 0x202A..0x202E, 0x2060..0x206F, 0x3164..0x3164, 0xFE00..0xFE0F, 0xFEFF..0xFEFF, - 0xFFA0..0xFFA0, 0xFFF0..0xFFF8, 0x1BCA0..0x1BCA3, 0x1D173..0x1D17A, 0xE0000..0xE0FFF, -) +// write-receipt.md: a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no Default_Ignorable code point anywhere, as of +// Unicode 16.0, compared raw (never trimmed). The tables are embedded (UnicodeMessageIdTables.kt, generated by +// scripts/gen-unicode-messageid-tables.py), not the JDK's, so the running JVM's Unicode version changes no result; the other +// four runtimes embed the same generated tables. +private fun inRanges(ranges: IntArray, point: Int): Boolean { + var low = 0 + var high = ranges.size / 2 - 1 + while (low <= high) { + val mid = (low + high) ushr 1 + when { + point < ranges[2 * mid] -> high = mid - 1 + point > ranges[2 * mid + 1] -> low = mid + 1 + else -> return true + } + } + return false +} -/** write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never trimmed). */ private fun admittedMessageId(message: CultNetMessage): String { val id = nonEmpty(message, "messageId") var hasLetterOrNumber = false id.codePoints().forEach { cp -> - if (DEFAULT_IGNORABLE.any { cp in it }) throw IOException("messageId holds a Default_Ignorable code point") - when (Character.getType(cp).toByte()) { - Character.UPPERCASE_LETTER, Character.LOWERCASE_LETTER, Character.TITLECASE_LETTER, Character.MODIFIER_LETTER, - Character.OTHER_LETTER, Character.DECIMAL_DIGIT_NUMBER, Character.LETTER_NUMBER, Character.OTHER_NUMBER -> hasLetterOrNumber = true - } + if (inRanges(DEFAULT_IGNORABLE, cp)) throw IOException("messageId holds a Default_Ignorable code point") + if (inRanges(LETTER_OR_NUMBER, cp)) hasLetterOrNumber = true } if (!hasLetterOrNumber) throw IOException("messageId must contain a letter or number") return id From c2d2c1d126a719bd2ff9a38929350dfff31a249e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:05:00 +0000 Subject: [PATCH 107/117] GameCult.Networking: write_admitted messageId from embedded Unicode 16.0 tables, not .NET's Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 47 +++--- .../CultNetUnicodeMessageIdTables.cs | 144 ++++++++++++++++++ 2 files changed, 164 insertions(+), 27 deletions(-) create mode 100644 src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index 9789e05e..3069384a 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -1,6 +1,5 @@ using System; using System.Buffers; -using System.Globalization; using MessagePack; namespace GameCult.Networking @@ -103,21 +102,13 @@ namespace GameCult.Networking } /// - /// The write_admitted messageId rule (write-receipt.md): a Unicode letter or number somewhere, no Default_Ignorable code - /// point anywhere, wire bytes that are valid UTF-8, compared raw (never trimmed). Letters and numbers come from the - /// runtime's Unicode categories (L*, Nd, Nl, No); a code point it does not assign (U+0378 in every version) has neither. - /// .NET has no Default_Ignorable property, so the ranges are embedded from DerivedCoreProperties.txt, Unicode 16.0 - /// (unchanged since 15.1); the Kotlin, Python and Rust decoders embed the same ranges, TypeScript uses its regex engine's. + /// The write_admitted messageId rule (write-receipt.md): a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no + /// Default_Ignorable code point anywhere, as of Unicode 16.0, wire bytes that are valid UTF-8, compared raw (never trimmed). + /// The tables are embedded (CultNetUnicodeMessageIdTables.cs, generated by scripts/gen-unicode-messageid-tables.py), not + /// .NET's, so the running runtime's Unicode version changes no result; the other four runtimes embed the same tables. /// internal static class CultNetAdmittedMessageId { - private static readonly (int Low, int High)[] DefaultIgnorable = - { - (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), - (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), - (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), - }; - private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); /// Throws unless the frame's top-level messageId is a valid write_admitted id; other keys are left to the serializer. @@ -148,23 +139,25 @@ namespace GameCult.Networking for (var at = 0; at < id.Length; at += char.IsSurrogatePair(id, at) ? 2 : 1) { var point = char.IsSurrogatePair(id, at) ? char.ConvertToUtf32(id, at) : id[at]; - if (Array.Exists(DefaultIgnorable, range => point >= range.Low && point <= range.High)) return false; - switch (CharUnicodeInfo.GetUnicodeCategory(point)) - { - case UnicodeCategory.UppercaseLetter: - case UnicodeCategory.LowercaseLetter: - case UnicodeCategory.TitlecaseLetter: - case UnicodeCategory.ModifierLetter: - case UnicodeCategory.OtherLetter: - case UnicodeCategory.DecimalDigitNumber: - case UnicodeCategory.LetterNumber: - case UnicodeCategory.OtherNumber: - hasLetterOrNumber = true; - break; - } + if (InRanges(CultNetUnicodeMessageIdTables.DefaultIgnorable, point)) return false; + hasLetterOrNumber |= InRanges(CultNetUnicodeMessageIdTables.LetterOrNumber, point); } return hasLetterOrNumber; } + + private static bool InRanges(int[] ranges, int point) + { + int low = 0, high = ranges.Length / 2 - 1; + while (low <= high) + { + var mid = (low + high) >> 1; + if (point < ranges[2 * mid]) high = mid - 1; + else if (point > ranges[2 * mid + 1]) low = mid + 1; + else return true; + } + + return false; + } } } diff --git a/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs b/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs new file mode 100644 index 00000000..45cedbc5 --- /dev/null +++ b/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs @@ -0,0 +1,144 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +namespace GameCult.Networking +{ + internal static class CultNetUnicodeMessageIdTables + { + // (low, high) pairs, flattened and sorted. + public static readonly int[] LetterOrNumber = + { + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, + 0xD6, 0xD8, 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, + 0x376, 0x377, 0x37A, 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, + 0x3F5, 0x3F7, 0x481, 0x48A, 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, + 0x620, 0x64A, 0x660, 0x669, 0x66E, 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, + 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, + 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, + 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, + 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, + 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, + 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, 0xA5C, 0xA5E, 0xA5E, 0xA66, + 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, 0xAB3, 0xAB5, 0xAB9, + 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, 0xB10, 0xB13, + 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, 0xB6F, + 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, + 0xC12, 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, + 0xC7E, 0xC80, 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, + 0xCDD, 0xCDE, 0xCE0, 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, + 0xD3D, 0xD4E, 0xD4E, 0xD54, 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, + 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, + 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, + 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, + 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, + 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, + 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, 0x124A, 0x124D, 0x1250, 0x1256, + 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, 0x12B2, 0x12B5, 0x12B8, + 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, 0x1318, 0x135A, + 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, 0x1681, + 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, + 0x1819, 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, + 0x1946, 0x196D, 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, + 0x1A54, 0x1A80, 0x1A89, 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, + 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, + 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, + 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, + 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, + 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, + 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, 0x2107, 0x2107, 0x210A, 0x2113, + 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, 0x212D, 0x212F, + 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, 0x24EA, 0x24FF, + 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, 0x2D27, + 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, + 0x2E2F, 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, + 0x30A1, 0x30FA, 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, + 0x31FF, 0x3220, 0x3229, 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, + 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, + 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, + 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, + 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, + 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, + 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, + 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, 0xAAE0, 0xAAEA, 0xAAF2, + 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, 0xAB30, 0xAB5A, + 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0xF900, + 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, + 0xFD8F, 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, + 0xFF41, 0xFF5A, 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, + 0x1000B, 0x1000D, 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, + 0x10080, 0x100FA, 0x10107, 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, + 0x102D0, 0x102E1, 0x102FB, 0x10300, 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, + 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, + 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, + 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, 0x105B1, 0x105B3, 0x105B9, 0x105BB, + 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, 0x10780, 0x10785, + 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, 0x10837, + 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, + 0x10A60, 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, + 0x10B35, 0x10B40, 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, + 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, + 0x10D85, 0x10E60, 0x10E7E, 0x10E80, 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, + 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, + 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, + 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, 0x11144, 0x11147, 0x11147, 0x11150, + 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, 0x111DA, 0x111DC, 0x111DC, + 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, 0x11286, 0x11288, + 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, 0x112F9, + 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, + 0x1138E, 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, + 0x11434, 0x11447, 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, + 0x114C7, 0x114C7, 0x114D0, 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, + 0x11644, 0x11650, 0x11659, 0x11680, 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, + 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, + 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, + 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D0, 0x119E1, 0x119E1, 0x119E3, + 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, 0x11A50, 0x11A5C, 0x11A89, + 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, 0x11C0A, + 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, 0x11D09, + 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, + 0x11F12, 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, + 0x1246E, 0x12480, 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, + 0x14400, 0x14646, 0x16100, 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, + 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, + 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, + 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, + 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, + 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, 0x1B152, 0x1B155, 0x1B155, + 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, 0x1BC88, 0x1BC90, + 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, + 0x1D516, 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, + 0x1D550, 0x1D552, 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, + 0x1D716, 0x1D734, 0x1D736, 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, + 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, + 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, + 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, + 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, + 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, 0x1ECAB, 0x1ECAD, 0x1ECAF, + 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, 0x1EE21, + 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, 0x1EE39, + 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, + 0x1EE5D, 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, + 0x1EE72, 0x1EE74, 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, + 0x1EEA1, 0x1EEA3, 0x1EEA5, 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, + 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, + 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, 0x323AF, + }; + + public static readonly int[] DefaultIgnorable = + { + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, + 0x202A, 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, + 0xFFF8, 0x1BCA0, 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, + }; + } +} From 8598fd3d12f588541fe5a2e34179cdb84e35c14f Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:05:00 +0000 Subject: [PATCH 108/117] cultnet-rs: write_admitted messageId from the generated Unicode 16.0 tables (letters and numbers now 16.0, not 15.1) Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-rs/src/contracts.rs | 139 +---------------- packages/cultnet-rs/src/lib.rs | 1 + .../src/unicode_messageid_tables.rs | 147 ++++++++++++++++++ 3 files changed, 155 insertions(+), 132 deletions(-) create mode 100644 packages/cultnet-rs/src/unicode_messageid_tables.rs diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 201632a2..bf907806 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -934,137 +934,12 @@ fn require_non_empty(value: &str, field: &str) -> Result<()> { Ok(()) } -// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (\p{L}) or number (\p{N}) and no -// Default_Ignorable_Code_Point, compared raw (never trimmed). std has `char::is_numeric` (N) but no letter category -// (`is_alphabetic` is the Alphabetic property, which also takes combining marks), and no Default_Ignorable, so both tables -// are embedded: DerivedCoreProperties.txt ranges, Unicode 16.0 (unchanged since 15.1), and the general category L ranges of -// Unicode 15.1 (U+0378 is unassigned in every version, so it is refused either way). The Kotlin, Python and C# decoders embed -// the same Default_Ignorable ranges and take L/N from their platform; TypeScript uses its regex engine's tables. -// Invalid UTF-8 never reaches this check: the MessagePack str decode refuses it. -const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ - (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), - (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), - (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), -]; - -const LETTERS: &[(u32, u32)] = &[ - (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB5, 0xB5), (0xBA, 0xBA), (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), - (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), - (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), - (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), - (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), (0x6E5, 0x6E6), (0x6EE, 0x6EF), (0x6FA, 0x6FC), (0x6FF, 0x6FF), - (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), (0x7CA, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), - (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), - (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), (0x950, 0x950), (0x958, 0x961), (0x971, 0x980), - (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), - (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), (0x9F0, 0x9F1), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), - (0xA13, 0xA28), (0xA2A, 0xA30), (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), - (0xA72, 0xA74), (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), - (0xABD, 0xABD), (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), - (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB71, 0xB71), - (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), (0xB9E, 0xB9F), - (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xC05, 0xC0C), (0xC0E, 0xC10), (0xC12, 0xC28), - (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), (0xC80, 0xC80), (0xC85, 0xC8C), - (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), - (0xCF1, 0xCF2), (0xD04, 0xD0C), (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), - (0xD5F, 0xD61), (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), - (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), (0xE8C, 0xEA3), - (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), (0xEDC, 0xEDF), - (0xF00, 0xF00), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), (0x1000, 0x102A), (0x103F, 0x103F), - (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), (0x106E, 0x1070), (0x1075, 0x1081), - (0x108E, 0x108E), (0x10A0, 0x10C5), (0x10C7, 0x10C7), (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), - (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), - (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), - (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), - (0x1401, 0x166C), (0x166F, 0x167F), (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16F1, 0x16F8), (0x1700, 0x1711), - (0x171F, 0x1731), (0x1740, 0x1751), (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), - (0x17DC, 0x17DC), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), (0x18B0, 0x18F5), - (0x1900, 0x191E), (0x1950, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), (0x1A00, 0x1A16), - (0x1A20, 0x1A54), (0x1AA7, 0x1AA7), (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B83, 0x1BA0), (0x1BAE, 0x1BAF), - (0x1BBA, 0x1BE5), (0x1C00, 0x1C23), (0x1C4D, 0x1C4F), (0x1C5A, 0x1C7D), (0x1C80, 0x1C88), (0x1C90, 0x1CBA), - (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), - (0x1E00, 0x1F15), (0x1F18, 0x1F1D), (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), - (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), - (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), - (0x1FF6, 0x1FFC), (0x2071, 0x2071), (0x207F, 0x207F), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), - (0x210A, 0x2113), (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), - (0x212A, 0x212D), (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2183, 0x2184), - (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), - (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), - (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), - (0x3005, 0x3006), (0x3031, 0x3035), (0x303B, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), - (0x30FC, 0x30FF), (0x3105, 0x312F), (0x3131, 0x318E), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3400, 0x4DBF), - (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), (0xA500, 0xA60C), (0xA610, 0xA61F), (0xA62A, 0xA62B), (0xA640, 0xA66E), - (0xA67F, 0xA69D), (0xA6A0, 0xA6E5), (0xA717, 0xA71F), (0xA722, 0xA788), (0xA78B, 0xA7CA), (0xA7D0, 0xA7D1), - (0xA7D3, 0xA7D3), (0xA7D5, 0xA7D9), (0xA7F2, 0xA801), (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), - (0xA840, 0xA873), (0xA882, 0xA8B3), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA90A, 0xA925), - (0xA930, 0xA946), (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9CF), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9EF), - (0xA9FA, 0xA9FE), (0xAA00, 0xAA28), (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), - (0xAA7E, 0xAAAF), (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), - (0xAADB, 0xAADD), (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), - (0xAB20, 0xAB26), (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xAC00, 0xD7A3), - (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), - (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), - (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), - (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), (0xFFC2, 0xFFC7), - (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), (0x10028, 0x1003A), - (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), (0x10280, 0x1029C), - (0x102A0, 0x102D0), (0x10300, 0x1031F), (0x1032D, 0x10340), (0x10342, 0x10349), (0x10350, 0x10375), - (0x10380, 0x1039D), (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x10400, 0x1049D), (0x104B0, 0x104D3), - (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), (0x1057C, 0x1058A), - (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), (0x105B3, 0x105B9), - (0x105BB, 0x105BC), (0x10600, 0x10736), (0x10740, 0x10755), (0x10760, 0x10767), (0x10780, 0x10785), - (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), (0x10808, 0x10808), (0x1080A, 0x10835), - (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), (0x10860, 0x10876), (0x10880, 0x1089E), - (0x108E0, 0x108F2), (0x108F4, 0x108F5), (0x10900, 0x10915), (0x10920, 0x10939), (0x10980, 0x109B7), - (0x109BE, 0x109BF), (0x10A00, 0x10A00), (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), - (0x10A60, 0x10A7C), (0x10A80, 0x10A9C), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10B00, 0x10B35), - (0x10B40, 0x10B55), (0x10B60, 0x10B72), (0x10B80, 0x10B91), (0x10C00, 0x10C48), (0x10C80, 0x10CB2), - (0x10CC0, 0x10CF2), (0x10D00, 0x10D23), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10F00, 0x10F1C), - (0x10F27, 0x10F27), (0x10F30, 0x10F45), (0x10F70, 0x10F81), (0x10FB0, 0x10FC4), (0x10FE0, 0x10FF6), - (0x11003, 0x11037), (0x11071, 0x11072), (0x11075, 0x11075), (0x11083, 0x110AF), (0x110D0, 0x110E8), - (0x11103, 0x11126), (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), - (0x11183, 0x111B2), (0x111C1, 0x111C4), (0x111DA, 0x111DA), (0x111DC, 0x111DC), (0x11200, 0x11211), - (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), - (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x11305, 0x1130C), (0x1130F, 0x11310), - (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), (0x1133D, 0x1133D), - (0x11350, 0x11350), (0x1135D, 0x11361), (0x11400, 0x11434), (0x11447, 0x1144A), (0x1145F, 0x11461), - (0x11480, 0x114AF), (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x11580, 0x115AE), (0x115D8, 0x115DB), - (0x11600, 0x1162F), (0x11644, 0x11644), (0x11680, 0x116AA), (0x116B8, 0x116B8), (0x11700, 0x1171A), - (0x11740, 0x11746), (0x11800, 0x1182B), (0x118A0, 0x118DF), (0x118FF, 0x11906), (0x11909, 0x11909), - (0x1190C, 0x11913), (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), - (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), - (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), - (0x11AB0, 0x11AF8), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), (0x11C40, 0x11C40), (0x11C72, 0x11C8F), - (0x11D00, 0x11D06), (0x11D08, 0x11D09), (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D60, 0x11D65), - (0x11D67, 0x11D68), (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), - (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11FB0, 0x11FB0), (0x12000, 0x12399), (0x12480, 0x12543), - (0x12F90, 0x12FF0), (0x13000, 0x1342F), (0x13441, 0x13446), (0x14400, 0x14646), (0x16800, 0x16A38), - (0x16A40, 0x16A5E), (0x16A70, 0x16ABE), (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), - (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16E40, 0x16E7F), (0x16F00, 0x16F4A), (0x16F50, 0x16F50), - (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), (0x17000, 0x187F7), (0x18800, 0x18CD5), - (0x18D00, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), - (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), - (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), (0x1BC90, 0x1BC99), (0x1D400, 0x1D454), - (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), (0x1D4A9, 0x1D4AC), - (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), (0x1D507, 0x1D50A), - (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), (0x1D540, 0x1D544), - (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), (0x1D6C2, 0x1D6DA), - (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), (0x1D750, 0x1D76E), - (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), (0x1DF00, 0x1DF1E), - (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), (0x1E137, 0x1E13D), (0x1E14E, 0x1E14E), - (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), (0x1E4D0, 0x1E4EB), (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), - (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), - (0x1EE00, 0x1EE03), (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), - (0x1EE29, 0x1EE32), (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), - (0x1EE47, 0x1EE47), (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), - (0x1EE54, 0x1EE54), (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), - (0x1EE5F, 0x1EE5F), (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), - (0x1EE74, 0x1EE77), (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), - (0x1EEA1, 0x1EEA3), (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x20000, 0x2A6DF), (0x2A700, 0x2B739), - (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), - (0x30000, 0x3134A), (0x31350, 0x323AF), -]; +// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) and no +// Default_Ignorable_Code_Point, as of Unicode 16.0, compared raw (never trimmed). The tables are embedded +// (unicode_messageid_tables.rs, generated by scripts/gen-unicode-messageid-tables.py), not std's, so no platform's Unicode +// version changes a result; the other four runtimes embed the same generated tables. Invalid UTF-8 never reaches this check: +// the MessagePack str decode refuses it. +use crate::unicode_messageid_tables::{DEFAULT_IGNORABLE, LETTER_OR_NUMBER}; fn in_ranges(ranges: &[(u32, u32)], point: u32) -> bool { ranges.binary_search_by(|&(low, high)| { @@ -1086,7 +961,7 @@ fn require_admitted_message_id(value: &str) -> Result<()> { if in_ranges(DEFAULT_IGNORABLE, point) { return Err(anyhow!("CultNet field messageId must not hold a Default_Ignorable code point")); } - has_letter_or_number |= ch.is_numeric() || in_ranges(LETTERS, point); + has_letter_or_number |= in_ranges(LETTER_OR_NUMBER, point); } if !has_letter_or_number { return Err(anyhow!("CultNet field messageId must contain a letter or number")); diff --git a/packages/cultnet-rs/src/lib.rs b/packages/cultnet-rs/src/lib.rs index b35bd701..e008a2ba 100644 --- a/packages/cultnet-rs/src/lib.rs +++ b/packages/cultnet-rs/src/lib.rs @@ -3,6 +3,7 @@ mod contracts; mod idunn_deployment_brake; mod idunn_lifecycle_brake; mod media_fec; +mod unicode_messageid_tables; mod media_stream_contracts; mod media_stream_wire; mod provider_health_contracts; diff --git a/packages/cultnet-rs/src/unicode_messageid_tables.rs b/packages/cultnet-rs/src/unicode_messageid_tables.rs new file mode 100644 index 00000000..b5293f7a --- /dev/null +++ b/packages/cultnet-rs/src/unicode_messageid_tables.rs @@ -0,0 +1,147 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +// (low, high) pairs, sorted. +pub const LETTER_OR_NUMBER: &[(u32, u32)] = &[ + (0x30, 0x39), (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB2, 0xB3), (0xB5, 0xB5), (0xB9, 0xBA), (0xBC, 0xBE), + (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), + (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), + (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), + (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), (0x660, 0x669), (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), + (0x6E5, 0x6E6), (0x6EE, 0x6FC), (0x6FF, 0x6FF), (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), + (0x7C0, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), + (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), + (0x950, 0x950), (0x958, 0x961), (0x966, 0x96F), (0x971, 0x980), (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), + (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), + (0x9E6, 0x9F1), (0x9F4, 0x9F9), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), (0xA13, 0xA28), (0xA2A, 0xA30), + (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), (0xA66, 0xA6F), (0xA72, 0xA74), + (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), (0xABD, 0xABD), + (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAE6, 0xAEF), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), + (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB66, 0xB6F), + (0xB71, 0xB77), (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), + (0xB9E, 0xB9F), (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xBE6, 0xBF2), (0xC05, 0xC0C), + (0xC0E, 0xC10), (0xC12, 0xC28), (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), + (0xC66, 0xC6F), (0xC78, 0xC7E), (0xC80, 0xC80), (0xC85, 0xC8C), (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), + (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), (0xCE6, 0xCEF), (0xCF1, 0xCF2), (0xD04, 0xD0C), + (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), (0xD58, 0xD61), (0xD66, 0xD78), + (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), (0xDE6, 0xDEF), + (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE50, 0xE59), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), + (0xE8C, 0xEA3), (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), + (0xED0, 0xED9), (0xEDC, 0xEDF), (0xF00, 0xF00), (0xF20, 0xF33), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), + (0x1000, 0x102A), (0x103F, 0x1049), (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), + (0x106E, 0x1070), (0x1075, 0x1081), (0x108E, 0x108E), (0x1090, 0x1099), (0x10A0, 0x10C5), (0x10C7, 0x10C7), + (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), + (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), + (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), + (0x1369, 0x137C), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), (0x1401, 0x166C), (0x166F, 0x167F), + (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16EE, 0x16F8), (0x1700, 0x1711), (0x171F, 0x1731), (0x1740, 0x1751), + (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), (0x17DC, 0x17DC), (0x17E0, 0x17E9), + (0x17F0, 0x17F9), (0x1810, 0x1819), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), + (0x18B0, 0x18F5), (0x1900, 0x191E), (0x1946, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), + (0x19D0, 0x19DA), (0x1A00, 0x1A16), (0x1A20, 0x1A54), (0x1A80, 0x1A89), (0x1A90, 0x1A99), (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B50, 0x1B59), (0x1B83, 0x1BA0), (0x1BAE, 0x1BE5), (0x1C00, 0x1C23), + (0x1C40, 0x1C49), (0x1C4D, 0x1C7D), (0x1C80, 0x1C8A), (0x1C90, 0x1CBA), (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), (0x1E00, 0x1F15), (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), (0x1FF6, 0x1FFC), (0x2070, 0x2071), + (0x2074, 0x2079), (0x207F, 0x2089), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), (0x210A, 0x2113), + (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), (0x212A, 0x212D), + (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2150, 0x2189), (0x2460, 0x249B), + (0x24EA, 0x24FF), (0x2776, 0x2793), (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2CFD, 0x2CFD), + (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), (0x3005, 0x3007), (0x3021, 0x3029), (0x3031, 0x3035), + (0x3038, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), (0x30FC, 0x30FF), (0x3105, 0x312F), + (0x3131, 0x318E), (0x3192, 0x3195), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3220, 0x3229), (0x3248, 0x324F), + (0x3251, 0x325F), (0x3280, 0x3289), (0x32B1, 0x32BF), (0x3400, 0x4DBF), (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), (0xA610, 0xA62B), (0xA640, 0xA66E), (0xA67F, 0xA69D), (0xA6A0, 0xA6EF), (0xA717, 0xA71F), + (0xA722, 0xA788), (0xA78B, 0xA7CD), (0xA7D0, 0xA7D1), (0xA7D3, 0xA7D3), (0xA7D5, 0xA7DC), (0xA7F2, 0xA801), + (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), (0xA830, 0xA835), (0xA840, 0xA873), (0xA882, 0xA8B3), + (0xA8D0, 0xA8D9), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA900, 0xA925), (0xA930, 0xA946), + (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9D9), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9FE), (0xAA00, 0xAA28), + (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA50, 0xAA59), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xABF0, 0xABF9), (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), + (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF10, 0xFF19), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), + (0x10028, 0x1003A), (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), + (0x10107, 0x10133), (0x10140, 0x10178), (0x1018A, 0x1018B), (0x10280, 0x1029C), (0x102A0, 0x102D0), + (0x102E1, 0x102FB), (0x10300, 0x10323), (0x1032D, 0x1034A), (0x10350, 0x10375), (0x10380, 0x1039D), + (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x103D1, 0x103D5), (0x10400, 0x1049D), (0x104A0, 0x104A9), + (0x104B0, 0x104D3), (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), + (0x1057C, 0x1058A), (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), + (0x105B3, 0x105B9), (0x105BB, 0x105BC), (0x105C0, 0x105F3), (0x10600, 0x10736), (0x10740, 0x10755), + (0x10760, 0x10767), (0x10780, 0x10785), (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), + (0x10808, 0x10808), (0x1080A, 0x10835), (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), + (0x10858, 0x10876), (0x10879, 0x1089E), (0x108A7, 0x108AF), (0x108E0, 0x108F2), (0x108F4, 0x108F5), + (0x108FB, 0x1091B), (0x10920, 0x10939), (0x10980, 0x109B7), (0x109BC, 0x109CF), (0x109D2, 0x10A00), + (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), (0x10A40, 0x10A48), (0x10A60, 0x10A7E), + (0x10A80, 0x10A9F), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10AEB, 0x10AEF), (0x10B00, 0x10B35), + (0x10B40, 0x10B55), (0x10B58, 0x10B72), (0x10B78, 0x10B91), (0x10BA9, 0x10BAF), (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), (0x10CC0, 0x10CF2), (0x10CFA, 0x10D23), (0x10D30, 0x10D39), (0x10D40, 0x10D65), + (0x10D6F, 0x10D85), (0x10E60, 0x10E7E), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10EC2, 0x10EC4), + (0x10F00, 0x10F27), (0x10F30, 0x10F45), (0x10F51, 0x10F54), (0x10F70, 0x10F81), (0x10FB0, 0x10FCB), + (0x10FE0, 0x10FF6), (0x11003, 0x11037), (0x11052, 0x1106F), (0x11071, 0x11072), (0x11075, 0x11075), + (0x11083, 0x110AF), (0x110D0, 0x110E8), (0x110F0, 0x110F9), (0x11103, 0x11126), (0x11136, 0x1113F), + (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), (0x11183, 0x111B2), + (0x111C1, 0x111C4), (0x111D0, 0x111DA), (0x111DC, 0x111DC), (0x111E1, 0x111F4), (0x11200, 0x11211), + (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), + (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x112F0, 0x112F9), (0x11305, 0x1130C), + (0x1130F, 0x11310), (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), + (0x1133D, 0x1133D), (0x11350, 0x11350), (0x1135D, 0x11361), (0x11380, 0x11389), (0x1138B, 0x1138B), + (0x1138E, 0x1138E), (0x11390, 0x113B5), (0x113B7, 0x113B7), (0x113D1, 0x113D1), (0x113D3, 0x113D3), + (0x11400, 0x11434), (0x11447, 0x1144A), (0x11450, 0x11459), (0x1145F, 0x11461), (0x11480, 0x114AF), + (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x114D0, 0x114D9), (0x11580, 0x115AE), (0x115D8, 0x115DB), + (0x11600, 0x1162F), (0x11644, 0x11644), (0x11650, 0x11659), (0x11680, 0x116AA), (0x116B8, 0x116B8), + (0x116C0, 0x116C9), (0x116D0, 0x116E3), (0x11700, 0x1171A), (0x11730, 0x1173B), (0x11740, 0x11746), + (0x11800, 0x1182B), (0x118A0, 0x118F2), (0x118FF, 0x11906), (0x11909, 0x11909), (0x1190C, 0x11913), + (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), (0x11950, 0x11959), + (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), (0x11BC0, 0x11BE0), (0x11BF0, 0x11BF9), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), (0x11C50, 0x11C6C), (0x11C72, 0x11C8F), (0x11D00, 0x11D06), (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D50, 0x11D59), (0x11D60, 0x11D65), (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11DA0, 0x11DA9), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), + (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11F50, 0x11F59), (0x11FB0, 0x11FB0), (0x11FC0, 0x11FD4), + (0x12000, 0x12399), (0x12400, 0x1246E), (0x12480, 0x12543), (0x12F90, 0x12FF0), (0x13000, 0x1342F), + (0x13441, 0x13446), (0x13460, 0x143FA), (0x14400, 0x14646), (0x16100, 0x1611D), (0x16130, 0x16139), + (0x16800, 0x16A38), (0x16A40, 0x16A5E), (0x16A60, 0x16A69), (0x16A70, 0x16ABE), (0x16AC0, 0x16AC9), + (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), (0x16B50, 0x16B59), (0x16B5B, 0x16B61), + (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16D40, 0x16D6C), (0x16D70, 0x16D79), (0x16E40, 0x16E96), + (0x16F00, 0x16F4A), (0x16F50, 0x16F50), (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), (0x18800, 0x18CD5), (0x18CFF, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), + (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), (0x1CCF0, 0x1CCF9), (0x1D2C0, 0x1D2D3), (0x1D2E0, 0x1D2F3), (0x1D360, 0x1D378), + (0x1D400, 0x1D454), (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), + (0x1D7CE, 0x1D7FF), (0x1DF00, 0x1DF1E), (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), (0x1E140, 0x1E149), (0x1E14E, 0x1E14E), (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), + (0x1E2F0, 0x1E2F9), (0x1E4D0, 0x1E4EB), (0x1E4F0, 0x1E4F9), (0x1E5D0, 0x1E5ED), (0x1E5F0, 0x1E5FA), + (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), + (0x1E8C7, 0x1E8CF), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), (0x1E950, 0x1E959), (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), (0x1ECB1, 0x1ECB4), (0x1ED01, 0x1ED2D), (0x1ED2F, 0x1ED3D), (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x1F100, 0x1F10C), (0x1FBF0, 0x1FBF9), (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), (0x30000, 0x3134A), (0x31350, 0x323AF), +]; + +pub const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ + (0xAD, 0xAD), (0x34F, 0x34F), (0x61C, 0x61C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +]; From f378d4fdb095dc7f902c243724d8d0f6faf18456 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:11:15 +0000 Subject: [PATCH 109/117] tests first: duplicate messageId, ill-formed id, scoped UTF-8 strictness, every table range pinned (failing) Failing before the fix: C# 1 (duplicate messageId, second is U+200B), TypeScript 1 (a + lone surrogate), Kotlin 1 (a document payload string with a lone surrogate no longer decodes). Python and Rust already pass the new vectors. The range probes read the generated write-admitted-message-id-ranges.json and pass on the correct tables; they exist to fail when any range is deleted or narrowed. Co-Authored-By: Claude Sonnet 5.5 --- .../write-admitted-message-id-ranges.json | 1 + .../interop/write-receipt-vectors.json | 122 +++++++++++++++++- .../gamecult/cultmesh/WriteReceiptVectors.kt | 32 +++++ .../cultnet-py/tests/test_write_receipt.py | 26 ++++ packages/cultnet-rs/tests/write_receipt.rs | 37 ++++++ .../cultnet-ts/test/write-receipt.test.ts | 20 +++ scripts/gen-unicode-messageid-tables.py | 14 +- .../CultNetWriteAdmittedMessageIdTests.cs | 33 +++++ 8 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 contracts/cultnet/interop/write-admitted-message-id-ranges.json diff --git a/contracts/cultnet/interop/write-admitted-message-id-ranges.json b/contracts/cultnet/interop/write-admitted-message-id-ranges.json new file mode 100644 index 00000000..98ccdec0 --- /dev/null +++ b/contracts/cultnet/interop/write-admitted-message-id-ranges.json @@ -0,0 +1 @@ +{"generated":"generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit","unicodeVersion":"16.0.0","letterOrNumber":[[48,57],[65,90],[97,122],[170,170],[178,179],[181,181],[185,186],[188,190],[192,214],[216,246],[248,705],[710,721],[736,740],[748,748],[750,750],[880,884],[886,887],[890,893],[895,895],[902,902],[904,906],[908,908],[910,929],[931,1013],[1015,1153],[1162,1327],[1329,1366],[1369,1369],[1376,1416],[1488,1514],[1519,1522],[1568,1610],[1632,1641],[1646,1647],[1649,1747],[1749,1749],[1765,1766],[1774,1788],[1791,1791],[1808,1808],[1810,1839],[1869,1957],[1969,1969],[1984,2026],[2036,2037],[2042,2042],[2048,2069],[2074,2074],[2084,2084],[2088,2088],[2112,2136],[2144,2154],[2160,2183],[2185,2190],[2208,2249],[2308,2361],[2365,2365],[2384,2384],[2392,2401],[2406,2415],[2417,2432],[2437,2444],[2447,2448],[2451,2472],[2474,2480],[2482,2482],[2486,2489],[2493,2493],[2510,2510],[2524,2525],[2527,2529],[2534,2545],[2548,2553],[2556,2556],[2565,2570],[2575,2576],[2579,2600],[2602,2608],[2610,2611],[2613,2614],[2616,2617],[2649,2652],[2654,2654],[2662,2671],[2674,2676],[2693,2701],[2703,2705],[2707,2728],[2730,2736],[2738,2739],[2741,2745],[2749,2749],[2768,2768],[2784,2785],[2790,2799],[2809,2809],[2821,2828],[2831,2832],[2835,2856],[2858,2864],[2866,2867],[2869,2873],[2877,2877],[2908,2909],[2911,2913],[2918,2927],[2929,2935],[2947,2947],[2949,2954],[2958,2960],[2962,2965],[2969,2970],[2972,2972],[2974,2975],[2979,2980],[2984,2986],[2990,3001],[3024,3024],[3046,3058],[3077,3084],[3086,3088],[3090,3112],[3114,3129],[3133,3133],[3160,3162],[3165,3165],[3168,3169],[3174,3183],[3192,3198],[3200,3200],[3205,3212],[3214,3216],[3218,3240],[3242,3251],[3253,3257],[3261,3261],[3293,3294],[3296,3297],[3302,3311],[3313,3314],[3332,3340],[3342,3344],[3346,3386],[3389,3389],[3406,3406],[3412,3414],[3416,3425],[3430,3448],[3450,3455],[3461,3478],[3482,3505],[3507,3515],[3517,3517],[3520,3526],[3558,3567],[3585,3632],[3634,3635],[3648,3654],[3664,3673],[3713,3714],[3716,3716],[3718,3722],[3724,3747],[3749,3749],[3751,3760],[3762,3763],[3773,3773],[3776,3780],[3782,3782],[3792,3801],[3804,3807],[3840,3840],[3872,3891],[3904,3911],[3913,3948],[3976,3980],[4096,4138],[4159,4169],[4176,4181],[4186,4189],[4193,4193],[4197,4198],[4206,4208],[4213,4225],[4238,4238],[4240,4249],[4256,4293],[4295,4295],[4301,4301],[4304,4346],[4348,4680],[4682,4685],[4688,4694],[4696,4696],[4698,4701],[4704,4744],[4746,4749],[4752,4784],[4786,4789],[4792,4798],[4800,4800],[4802,4805],[4808,4822],[4824,4880],[4882,4885],[4888,4954],[4969,4988],[4992,5007],[5024,5109],[5112,5117],[5121,5740],[5743,5759],[5761,5786],[5792,5866],[5870,5880],[5888,5905],[5919,5937],[5952,5969],[5984,5996],[5998,6000],[6016,6067],[6103,6103],[6108,6108],[6112,6121],[6128,6137],[6160,6169],[6176,6264],[6272,6276],[6279,6312],[6314,6314],[6320,6389],[6400,6430],[6470,6509],[6512,6516],[6528,6571],[6576,6601],[6608,6618],[6656,6678],[6688,6740],[6784,6793],[6800,6809],[6823,6823],[6917,6963],[6981,6988],[6992,7001],[7043,7072],[7086,7141],[7168,7203],[7232,7241],[7245,7293],[7296,7306],[7312,7354],[7357,7359],[7401,7404],[7406,7411],[7413,7414],[7418,7418],[7424,7615],[7680,7957],[7960,7965],[7968,8005],[8008,8013],[8016,8023],[8025,8025],[8027,8027],[8029,8029],[8031,8061],[8064,8116],[8118,8124],[8126,8126],[8130,8132],[8134,8140],[8144,8147],[8150,8155],[8160,8172],[8178,8180],[8182,8188],[8304,8305],[8308,8313],[8319,8329],[8336,8348],[8450,8450],[8455,8455],[8458,8467],[8469,8469],[8473,8477],[8484,8484],[8486,8486],[8488,8488],[8490,8493],[8495,8505],[8508,8511],[8517,8521],[8526,8526],[8528,8585],[9312,9371],[9450,9471],[10102,10131],[11264,11492],[11499,11502],[11506,11507],[11517,11517],[11520,11557],[11559,11559],[11565,11565],[11568,11623],[11631,11631],[11648,11670],[11680,11686],[11688,11694],[11696,11702],[11704,11710],[11712,11718],[11720,11726],[11728,11734],[11736,11742],[11823,11823],[12293,12295],[12321,12329],[12337,12341],[12344,12348],[12353,12438],[12445,12447],[12449,12538],[12540,12543],[12549,12591],[12593,12686],[12690,12693],[12704,12735],[12784,12799],[12832,12841],[12872,12879],[12881,12895],[12928,12937],[12977,12991],[13312,19903],[19968,42124],[42192,42237],[42240,42508],[42512,42539],[42560,42606],[42623,42653],[42656,42735],[42775,42783],[42786,42888],[42891,42957],[42960,42961],[42963,42963],[42965,42972],[42994,43009],[43011,43013],[43015,43018],[43020,43042],[43056,43061],[43072,43123],[43138,43187],[43216,43225],[43250,43255],[43259,43259],[43261,43262],[43264,43301],[43312,43334],[43360,43388],[43396,43442],[43471,43481],[43488,43492],[43494,43518],[43520,43560],[43584,43586],[43588,43595],[43600,43609],[43616,43638],[43642,43642],[43646,43695],[43697,43697],[43701,43702],[43705,43709],[43712,43712],[43714,43714],[43739,43741],[43744,43754],[43762,43764],[43777,43782],[43785,43790],[43793,43798],[43808,43814],[43816,43822],[43824,43866],[43868,43881],[43888,44002],[44016,44025],[44032,55203],[55216,55238],[55243,55291],[63744,64109],[64112,64217],[64256,64262],[64275,64279],[64285,64285],[64287,64296],[64298,64310],[64312,64316],[64318,64318],[64320,64321],[64323,64324],[64326,64433],[64467,64829],[64848,64911],[64914,64967],[65008,65019],[65136,65140],[65142,65276],[65296,65305],[65313,65338],[65345,65370],[65382,65470],[65474,65479],[65482,65487],[65490,65495],[65498,65500],[65536,65547],[65549,65574],[65576,65594],[65596,65597],[65599,65613],[65616,65629],[65664,65786],[65799,65843],[65856,65912],[65930,65931],[66176,66204],[66208,66256],[66273,66299],[66304,66339],[66349,66378],[66384,66421],[66432,66461],[66464,66499],[66504,66511],[66513,66517],[66560,66717],[66720,66729],[66736,66771],[66776,66811],[66816,66855],[66864,66915],[66928,66938],[66940,66954],[66956,66962],[66964,66965],[66967,66977],[66979,66993],[66995,67001],[67003,67004],[67008,67059],[67072,67382],[67392,67413],[67424,67431],[67456,67461],[67463,67504],[67506,67514],[67584,67589],[67592,67592],[67594,67637],[67639,67640],[67644,67644],[67647,67669],[67672,67702],[67705,67742],[67751,67759],[67808,67826],[67828,67829],[67835,67867],[67872,67897],[67968,68023],[68028,68047],[68050,68096],[68112,68115],[68117,68119],[68121,68149],[68160,68168],[68192,68222],[68224,68255],[68288,68295],[68297,68324],[68331,68335],[68352,68405],[68416,68437],[68440,68466],[68472,68497],[68521,68527],[68608,68680],[68736,68786],[68800,68850],[68858,68899],[68912,68921],[68928,68965],[68975,68997],[69216,69246],[69248,69289],[69296,69297],[69314,69316],[69376,69415],[69424,69445],[69457,69460],[69488,69505],[69552,69579],[69600,69622],[69635,69687],[69714,69743],[69745,69746],[69749,69749],[69763,69807],[69840,69864],[69872,69881],[69891,69926],[69942,69951],[69956,69956],[69959,69959],[69968,70002],[70006,70006],[70019,70066],[70081,70084],[70096,70106],[70108,70108],[70113,70132],[70144,70161],[70163,70187],[70207,70208],[70272,70278],[70280,70280],[70282,70285],[70287,70301],[70303,70312],[70320,70366],[70384,70393],[70405,70412],[70415,70416],[70419,70440],[70442,70448],[70450,70451],[70453,70457],[70461,70461],[70480,70480],[70493,70497],[70528,70537],[70539,70539],[70542,70542],[70544,70581],[70583,70583],[70609,70609],[70611,70611],[70656,70708],[70727,70730],[70736,70745],[70751,70753],[70784,70831],[70852,70853],[70855,70855],[70864,70873],[71040,71086],[71128,71131],[71168,71215],[71236,71236],[71248,71257],[71296,71338],[71352,71352],[71360,71369],[71376,71395],[71424,71450],[71472,71483],[71488,71494],[71680,71723],[71840,71922],[71935,71942],[71945,71945],[71948,71955],[71957,71958],[71960,71983],[71999,71999],[72001,72001],[72016,72025],[72096,72103],[72106,72144],[72161,72161],[72163,72163],[72192,72192],[72203,72242],[72250,72250],[72272,72272],[72284,72329],[72349,72349],[72368,72440],[72640,72672],[72688,72697],[72704,72712],[72714,72750],[72768,72768],[72784,72812],[72818,72847],[72960,72966],[72968,72969],[72971,73008],[73030,73030],[73040,73049],[73056,73061],[73063,73064],[73066,73097],[73112,73112],[73120,73129],[73440,73458],[73474,73474],[73476,73488],[73490,73523],[73552,73561],[73648,73648],[73664,73684],[73728,74649],[74752,74862],[74880,75075],[77712,77808],[77824,78895],[78913,78918],[78944,82938],[82944,83526],[90368,90397],[90416,90425],[92160,92728],[92736,92766],[92768,92777],[92784,92862],[92864,92873],[92880,92909],[92928,92975],[92992,92995],[93008,93017],[93019,93025],[93027,93047],[93053,93071],[93504,93548],[93552,93561],[93760,93846],[93952,94026],[94032,94032],[94099,94111],[94176,94177],[94179,94179],[94208,100343],[100352,101589],[101631,101640],[110576,110579],[110581,110587],[110589,110590],[110592,110882],[110898,110898],[110928,110930],[110933,110933],[110948,110951],[110960,111355],[113664,113770],[113776,113788],[113792,113800],[113808,113817],[118000,118009],[119488,119507],[119520,119539],[119648,119672],[119808,119892],[119894,119964],[119966,119967],[119970,119970],[119973,119974],[119977,119980],[119982,119993],[119995,119995],[119997,120003],[120005,120069],[120071,120074],[120077,120084],[120086,120092],[120094,120121],[120123,120126],[120128,120132],[120134,120134],[120138,120144],[120146,120485],[120488,120512],[120514,120538],[120540,120570],[120572,120596],[120598,120628],[120630,120654],[120656,120686],[120688,120712],[120714,120744],[120746,120770],[120772,120779],[120782,120831],[122624,122654],[122661,122666],[122928,122989],[123136,123180],[123191,123197],[123200,123209],[123214,123214],[123536,123565],[123584,123627],[123632,123641],[124112,124139],[124144,124153],[124368,124397],[124400,124410],[124896,124902],[124904,124907],[124909,124910],[124912,124926],[124928,125124],[125127,125135],[125184,125251],[125259,125259],[125264,125273],[126065,126123],[126125,126127],[126129,126132],[126209,126253],[126255,126269],[126464,126467],[126469,126495],[126497,126498],[126500,126500],[126503,126503],[126505,126514],[126516,126519],[126521,126521],[126523,126523],[126530,126530],[126535,126535],[126537,126537],[126539,126539],[126541,126543],[126545,126546],[126548,126548],[126551,126551],[126553,126553],[126555,126555],[126557,126557],[126559,126559],[126561,126562],[126564,126564],[126567,126570],[126572,126578],[126580,126583],[126585,126588],[126590,126590],[126592,126601],[126603,126619],[126625,126627],[126629,126633],[126635,126651],[127232,127244],[130032,130041],[131072,173791],[173824,177977],[177984,178205],[178208,183969],[183984,191456],[191472,192093],[194560,195101],[196608,201546],[201552,205743]],"defaultIgnorable":[[173,173],[847,847],[1564,1564],[4447,4448],[6068,6069],[6155,6159],[8203,8207],[8234,8238],[8288,8303],[12644,12644],[65024,65039],[65279,65279],[65440,65440],[65520,65528],[113824,113827],[119155,119162],[917504,921599]]} diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index c7f47738..f99265ed 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -150,7 +150,7 @@ ] }, "admittedMessageId": { - "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Letters, numbers and Default_Ignorable are those of Unicode 16.0 in every runtime, from embedded tables (no platform category lookup), so a runtime's own Unicode version cannot change a result. Scope: write_admitted only; error.v1 is untouched.", + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. Every messageId key of a frame must be valid. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string or this file cannot hold (a lone surrogate is not valid UTF-8; the bytes are ED A0 80) or a frame that repeats a key. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses earlier; byte-level strictness is a property of the reader and is not required of a decoder. Scope: write_admitted only; error.v1 is untouched.", "cases": [ { "name": "refuse_braille_blank_2800", @@ -222,7 +222,7 @@ "D800" ], "expect": "refuse", - "why": "not a Unicode scalar value; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str, so the decoder must refuse the frame. Rust, C#, Kotlin and Python build the frame from wireHex; a runtime whose strings can hold a lone surrogate (TypeScript) may also build it from the code point", + "why": "a lone surrogate is not a Unicode scalar value and has no letter or number; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str. A reader that rejects invalid UTF-8 refuses the frame; one that substitutes or keeps the surrogate refuses the id", "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" }, { @@ -408,6 +408,122 @@ "expect": "refuse", "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0" }, + { + "name": "refuse_duplicate_message_id_second_is_default_ignorable", + "codePoints": ["0061"], + "expect": "refuse", + "why": "the frame carries messageId twice, \"a\" and then U+200B; every messageId key must be valid, and a reader that keeps the last one sees U+200B", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B" + }, + { + "name": "refuse_letter_then_lone_surrogate_D800", + "codePoints": ["0061"], + "expect": "refuse", + "why": "\"a\" followed by a lone surrogate: the id as decoded is not well-formed Unicode. A reader that rejects invalid UTF-8 refuses the frame earlier; one that substitutes or keeps the surrogate must still refuse the id. Byte-level strictness is a property of the reader, not required of the decoder", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A461EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "accept_astral_mathematical_bold_capital_a_1D400", + "codePoints": ["1D400"], + "expect": "accept", + "why": "an astral letter (Lu)" + }, + { + "name": "accept_astral_cjk_ext_b_20000", + "codePoints": ["20000"], + "expect": "accept", + "why": "an astral letter (Lo)" + }, + { + "name": "refuse_letter_then_default_ignorable_061C", + "codePoints": ["0061", "061C"], + "expect": "refuse", + "why": "U+061C is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B4", + "codePoints": ["0061", "17B4"], + "expect": "refuse", + "why": "U+17B4 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B5", + "codePoints": ["0061", "17B5"], + "expect": "refuse", + "why": "U+17B5 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202A", + "codePoints": ["0061", "202A"], + "expect": "refuse", + "why": "U+202A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202E", + "codePoints": ["0061", "202E"], + "expect": "refuse", + "why": "U+202E is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE00", + "codePoints": ["0061", "FE00"], + "expect": "refuse", + "why": "U+FE00 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE0F", + "codePoints": ["0061", "FE0F"], + "expect": "refuse", + "why": "U+FE0F is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF0", + "codePoints": ["0061", "FFF0"], + "expect": "refuse", + "why": "U+FFF0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF8", + "codePoints": ["0061", "FFF8"], + "expect": "refuse", + "why": "U+FFF8 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA0", + "codePoints": ["0061", "1BCA0"], + "expect": "refuse", + "why": "U+1BCA0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA3", + "codePoints": ["0061", "1BCA3"], + "expect": "refuse", + "why": "U+1BCA3 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D173", + "codePoints": ["0061", "1D173"], + "expect": "refuse", + "why": "U+1D173 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D17A", + "codePoints": ["0061", "1D17A"], + "expect": "refuse", + "why": "U+1D17A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0000", + "codePoints": ["0061", "E0000"], + "expect": "refuse", + "why": "U+E0000 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0FFF", + "codePoints": ["0061", "E0FFF"], + "expect": "refuse", + "why": "U+E0FFF is Default_Ignorable (an end of one of its ranges)" + }, { "name": "accept_abc", "codePoints": [ @@ -1524,7 +1640,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 40 + "admittedMessageIdCases": 59 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index 8955c52d..35e0136f 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -68,6 +68,38 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every + // range, so deleting or narrowing any range fails here. + check("unicodeTableRanges") { + @Suppress("UNCHECKED_CAST") + val ranges = JsonReader(File(File(path).parentFile, "write-admitted-message-id-ranges.json").readText(Charsets.UTF_8)).readDocument() as Map + fun pairs(key: String) = list(ranges[key]).map { list(it).map { n -> (n as Long).toInt() } } + val letters = pairs("letterOrNumber") + val ignorable = pairs("defaultIgnorable") + val problems = mutableListOf() + if (letters.size <= 700 || ignorable.size != 17) problems += "ranges file holds ${letters.size} and ${ignorable.size} ranges" + fun accepted(id: String): Boolean = try { + decodeReceiptFrame(parseCultNetMessage(MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray())) + true + } catch (_: IOException) { false } + for ((low, high) in letters) for ((point, want) in listOf(low to true, high to true, low - 1 to false, high + 1 to false)) { + if (accepted(String(Character.toChars(point))) != want) problems += "U+${point.toString(16)} alone should be ${if (want) "accepted" else "refused"}" + } + for ((low, high) in ignorable) for ((point, want) in listOf(low to false, high to false, low - 1 to true, high + 1 to true)) { + if (accepted("a" + String(Character.toChars(point))) != want) problems += "a + U+${point.toString(16)} should be ${if (want) "accepted" else "refused"}" + } + problems + } + + // A document payload string with a lone surrogate (ED A0 80) is not a write_admitted id: the shared MessagePack reader keeps + // reading it. Strictness is scoped to the write_admitted frame, never to the reader. + check("documentPayloadStringWithLoneSurrogateStillDecodes") { + val text = MessagePackReader(unhex("A3EDA080")).readString() + if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string") + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 0c42a6af..fce0d850 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -94,6 +94,32 @@ class WriteReceiptVectorTests(unittest.TestCase): failures.append(f"{case['name']}: expected {case['expect']}") self.assertEqual(failures, []) + def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None: + # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and + # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and + # last point is refused and beside the points either side accepted. Deleting or narrowing any embedded range fails here. + ranges = json.loads((_INTEROP / "write-admitted-message-id-ranges.json").read_text(encoding="utf-8")) + self.assertGreater(len(ranges["letterOrNumber"]), 700) + self.assertEqual(len(ranges["defaultIgnorable"]), 17) + + def accepted(text: str) -> bool: + try: + parse_message(_pack(_admitted(messageId=text))) + return True + except ValueError: + return False + + failures = [] + for low, high in ranges["letterOrNumber"]: + for point, want in ((low, True), (high, True), (low - 1, False), (high + 1, False)): + if accepted(chr(point)) != want: + failures.append(f"U+{point:04X} alone should be {'accepted' if want else 'refused'}") + for low, high in ranges["defaultIgnorable"]: + for point, want in ((low, False), (high, False), (low - 1, True), (high + 1, True)): + if accepted("a" + chr(point)) != want: + failures.append(f"a + U+{point:04X} should be {'accepted' if want else 'refused'}") + self.assertEqual(failures, []) + def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: payload = None if case["payloadHex"] is None else bytes.fromhex(case["payloadHex"]) diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 8dccf996..10d7dcb8 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -201,3 +201,40 @@ fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() { } assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } + +/// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +#[test] +fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-admitted-message-id-ranges.json"); + let ranges: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let letters = ranges["letterOrNumber"].as_array().unwrap(); + let ignorable = ranges["defaultIgnorable"].as_array().unwrap(); + assert!(letters.len() > 700); + assert_eq!(ignorable.len(), 17); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let accepted = |id: &str| { + let header = vec![0xA0 | id.len() as u8]; + let frame = [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat(); + decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0).is_ok() + }; + let point = |value: i64| char::from_u32(value as u32).unwrap().to_string(); + let mut failures = Vec::new(); + for range in letters { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, true), (high, true), (low - 1, false), (high + 1, false)] { + if accepted(&point(at)) != want { + failures.push(format!("U+{at:X} alone should be {}", if want { "accepted" } else { "refused" })); + } + } + } + for range in ignorable { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, false), (high, false), (low - 1, true), (high + 1, true)] { + if accepted(&format!("a{}", point(at))) != want { + failures.push(format!("a + U+{at:X} should be {}", if want { "accepted" } else { "refused" })); + } + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index c447367a..c654de1a 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -303,3 +303,23 @@ test("write_admitted messageId: every admittedMessageId vector is refused or acc } assert.deepEqual(failures, []); }); + +// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned at both ends", () => { + const ranges = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"), "utf8")); + assert.ok(ranges.letterOrNumber.length > 700); + assert.equal(ranges.defaultIgnorable.length, 17); + const accepted = (id: string) => { try { parseCultNetMessage(admitted(id)); return true; } catch { return false; } }; + const failures: string[] = []; + for (const [low, high] of ranges.letterOrNumber as [number, number][]) { + for (const [point, want] of [[low, true], [high, true], [low - 1, false], [high + 1, false]] as [number, boolean][]) { + if (accepted(String.fromCodePoint(point)) !== want) failures.push(`U+${point.toString(16)} alone should be ${want ? "accepted" : "refused"}`); + } + } + for (const [low, high] of ranges.defaultIgnorable as [number, number][]) { + for (const [point, want] of [[low, false], [high, false], [low - 1, true], [high + 1, true]] as [number, boolean][]) { + if (accepted("a" + String.fromCodePoint(point)) !== want) failures.push(`a + U+${point.toString(16)} should be ${want ? "accepted" : "refused"}`); + } + } + assert.deepEqual(failures, []); +}); diff --git a/scripts/gen-unicode-messageid-tables.py b/scripts/gen-unicode-messageid-tables.py index 96d1a205..4c119930 100644 --- a/scripts/gen-unicode-messageid-tables.py +++ b/scripts/gen-unicode-messageid-tables.py @@ -13,6 +13,7 @@ https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --chec any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only. """ import hashlib +import json import re import sys from pathlib import Path @@ -28,6 +29,9 @@ TARGETS = { "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs", "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs", } +# The independent copy the runtimes' tests probe (first, last, and the code point either side of every range), so that deleting +# or narrowing any range of any embedded table fails a test. +RANGES_JSON = ROOT / "contracts/cultnet/interop/write-admitted-message-id-ranges.json" def parse(path, wanted): @@ -95,8 +99,12 @@ def main(): digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16] header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit" stale = [] - for target, path in TARGETS.items(): - text = emit(target, letters, ignorable, header) + texts = {path: emit(target, letters, ignorable, header) for target, path in TARGETS.items()} + texts[RANGES_JSON] = json.dumps( + {"generated": header, "unicodeVersion": VERSION, "letterOrNumber": [list(r) for r in letters], "defaultIgnorable": [list(r) for r in ignorable]}, + separators=(",", ":"), + ) + "\n" + for path, text in texts.items(): if check: if not path.exists() or path.read_text(encoding="utf-8") != text: stale.append(str(path.relative_to(ROOT))) @@ -104,7 +112,7 @@ def main(): path.write_text(text, encoding="utf-8") if stale: sys.exit("stale or hand-edited: " + ", ".join(stale)) - print(f"{'checked' if check else 'wrote'} {len(TARGETS)} tables: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") + print(f"{'checked' if check else 'wrote'} {len(texts)} files: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") if __name__ == "__main__": diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index f5cac528..a4f1475c 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -54,5 +54,38 @@ namespace GameCult.Networking.Tests Assert.That(decoded!.MessageId, Is.EqualTo(id)); } } + + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. + [Test] + public void EveryRangeOfTheUnicodeTablesIsPinnedAtBothEnds() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var ranges = JsonDocument.Parse(File.ReadAllText(Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"))).RootElement; + var letters = ranges.GetProperty("letterOrNumber").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + var ignorable = ranges.GetProperty("defaultIgnorable").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + Assert.That(letters.Length, Is.GreaterThan(700)); + Assert.That(ignorable.Length, Is.EqualTo(17)); + static bool Accepted(string id) + { + try + { + CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize( + new CultNetWriteAdmittedMessage { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" })); + return true; + } + catch (Exception) { return false; } + } + + var failures = new List(); + foreach (var (low, high) in letters) + foreach (var (point, want) in new[] { (low, true), (high, true), (low - 1, false), (high + 1, false) }) + if (Accepted(char.ConvertFromUtf32(point)) != want) failures.Add($"U+{point:X} alone should be {(want ? "accepted" : "refused")}"); + foreach (var (low, high) in ignorable) + foreach (var (point, want) in new[] { (low, false), (high, false), (low - 1, true), (high + 1, true) }) + if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}"); + Assert.That(failures, Is.Empty); + } } } From 2f372eb2f1658696f7115474bd4d7777246009fc Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:11:53 +0000 Subject: [PATCH 110/117] GameCult.Networking: check every messageId key of a write_admitted frame, not only the first Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetSchemaMessageSerialization.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index 3069384a..cb33b707 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -111,7 +111,7 @@ namespace GameCult.Networking { private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); - /// Throws unless the frame's top-level messageId is a valid write_admitted id; other keys are left to the serializer. + /// Throws unless every top-level messageId key of the frame is a valid write_admitted id; other keys are left to the serializer. public static void Require(byte[] payload) { var reader = new MessagePackReader(payload); @@ -125,7 +125,7 @@ namespace GameCult.Networking try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); - return; + continue; // a repeated messageId key is checked too: the serializer keeps the last, so every one must be valid } if (key == null) reader.Skip(); From 99273f468c5441ba67fa527c5834e3c2264ff7aa Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:12:07 +0000 Subject: [PATCH 111/117] cultnet-ts: refuse a write_admitted messageId that is not well-formed Unicode (lone surrogate) Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 09d5d823..8dd37ced 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -1478,7 +1478,8 @@ function isValidAdmittedMessageId(value: unknown): boolean { let hasLetterOrNumber = false; for (const ch of value) { const point = ch.codePointAt(0) as number; - if (inRanges(DEFAULT_IGNORABLE, point)) { + // The decoded string must be well-formed Unicode: @msgpack/msgpack turns invalid UTF-8 (ED A0 80) into a lone surrogate. + if ((point >= 0xd800 && point <= 0xdfff) || inRanges(DEFAULT_IGNORABLE, point)) { return false; } hasLetterOrNumber = hasLetterOrNumber || inRanges(LETTER_OR_NUMBER, point); From 6c130581a1193a810584a18b279d6f1fdc51c3f7 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:13:01 +0000 Subject: [PATCH 112/117] cultmesh-kotlin: UTF-8 strictness scoped to the write_admitted frame, not the shared MessagePack reader Co-Authored-By: Claude Sonnet 5.5 --- .../src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 2337448d..72c1d41d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -2365,6 +2365,9 @@ fun parseCultNetMessage(payload: ByteArray): CultNetMessage { if (schemaVersion !is String || schemaVersion.isBlank()) { throw IOException("CultNet schema-v0 messages must declare schemaVersion") } + // A write_admitted id is echoed byte for byte, so that frame alone is read again with invalid UTF-8 refused; document payload + // strings elsewhere keep decoding as before. + if (schemaVersion == WRITE_ADMITTED_VERSION) MessagePackReader(payload, strictUtf8 = true).readAny() val body = linkedMapOf() for ((key, value) in decoded) { if (key !is String) throw IOException("CultNet schema-v0 message keys must be strings") @@ -6152,7 +6155,8 @@ private fun requireSchemaResponse(message: CultNetMessage?, expectedSchemaVersio return response } -class MessagePackReader(payload: ByteArray) { +/** [strictUtf8] makes a str whose bytes are not UTF-8 throw instead of becoming U+FFFD; only the write_admitted frame asks for it. */ +class MessagePackReader(payload: ByteArray, private val strictUtf8: Boolean = false) { private val input = DataInputStream(ByteArrayInputStream(payload)) private var pushed = -1 @@ -6307,9 +6311,10 @@ class MessagePackReader(payload: ByteArray) { code == 0xdb -> input.readInt() else -> throw IOException("expected string") } - // Strict: bytes that are not UTF-8 (a lone surrogate's ED A0 80) refuse the frame instead of becoming U+FFFD. + val bytes = readPayload(length) + if (!strictUtf8) return String(bytes, StandardCharsets.UTF_8) return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(readPayload(length))).toString() + .decode(ByteBuffer.wrap(bytes)).toString() } private fun readBinaryAfterCode(code: Int): ByteArray { From a97861945c914bb2d08cc95174aa65d2366f196c Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:13:09 +0000 Subject: [PATCH 113/117] write-receipt.md: the messageId rule is on the decoded string; byte-level UTF-8 strictness is the reader's, not required Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 10f42a6a..4f6cb7d1 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. Every `messageId` key of a frame must be valid. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. @@ -67,7 +67,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; the lone surrogate U+D800 is carried as a whole frame in `wireHex`, because it is not valid UTF-8 and only some runtimes' strings can hold it), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; a lone surrogate, and a frame that repeats the key, are carried as whole frames in `wireHex`, because a lone surrogate is not valid UTF-8 and only some runtimes' strings can hold it; `write-admitted-message-id-ranges.json` lists every range of the embedded tables, and each runtime probes the first and last point of each range and the points either side), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance status From cc5a9cec9b446996859212625a0413b6cdd88b10 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:04:58 +0000 Subject: [PATCH 114/117] tests first: repeated messageId key, each runtime's declared policy (C# failing) Vectors admittedMessageIdDuplicates carry [invalid,valid], [valid,invalid], [valid,valid]. Decoders that see the duplicate (C#, Rust) must refuse all three; map-collapsing decoders (Python, TS, Kotlin) judge the last key. Python, TS, Kotlin and Rust already behave as declared; C# fails: it accepted [valid,valid]. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 28 ++++++++++++++++++- .../gamecult/cultmesh/WriteReceiptVectors.kt | 14 ++++++++++ .../cultnet-py/tests/test_write_receipt.py | 14 +++++++++- packages/cultnet-rs/tests/write_receipt.rs | 14 ++++++++++ .../cultnet-ts/test/write-receipt.test.ts | 12 ++++++++ .../CultNetWriteAdmittedMessageIdTests.cs | 14 ++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index f99265ed..e95c3180 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -606,6 +606,31 @@ } ] }, + "admittedMessageIdDuplicates": { + "semantics": "A frame with more than one messageId key is malformed. A decoder that can see the duplicate (C#, Rust: raw MessagePack reader or ordered pairs) refuses it, whatever the values (`seesDuplicate`). A decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses exactly when that value is invalid (`lastKeyWins`); where it accepts it echoes the last key's id (`lastKeyWinsEcho`). Neither behaviour ever echoes an invalid id. Each runtime declares which policy it has and its runner asserts that policy's column, so a change of a decoder's key policy fails a named test. The first key is ZWSP (U+200B, Default_Ignorable) in `invalid_then_valid`; the second key is the last entry of the map. Frames are MessagePack hex.", + "cases": [ + { + "name": "invalid_then_valid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3E2808BA96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A161", + "seesDuplicate": "refuse", + "lastKeyWins": "accept", + "lastKeyWinsEcho": "a" + }, + { + "name": "valid_then_invalid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A161A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B", + "seesDuplicate": "refuse", + "lastKeyWins": "refuse" + }, + { + "name": "valid_then_valid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A161A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A162", + "seesDuplicate": "refuse", + "lastKeyWins": "accept", + "lastKeyWinsEcho": "b" + } + ] + }, "schemaInvalid": [ { "name": "write_admitted_with_an_error_key", @@ -1640,7 +1665,8 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 59 + "admittedMessageIdCases": 59, + "admittedMessageIdDuplicateCases": 3 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index 35e0136f..b7215a12 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -100,6 +100,20 @@ fun replayWriteReceiptVectors(path: String): Boolean { if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string") } + // admittedMessageIdDuplicates: Kotlin decodes the frame to a map, so it cannot see a repeated key: it judges the last one. + for (case in list(obj(vectors["admittedMessageIdDuplicates"])["cases"])) { + val c = obj(case) + check("admittedMessageIdDuplicates/${c["name"]}") { + val decoded = try { decodeReceiptFrame(parseCultNetMessage(unhex(c["wireHex"] as String))) } catch (_: IOException) { null } + when { + c["lastKeyWins"] == "refuse" -> if (decoded == null) emptyList() else listOf("accepted a frame whose last messageId key is invalid") + decoded == null -> listOf("refused a frame whose last messageId key is valid") + decoded.toWireMap()["messageId"] != c["lastKeyWinsEcho"] -> listOf("echoed ${decoded.toWireMap()["messageId"]}, wanted ${c["lastKeyWinsEcho"]}") + else -> emptyList() + } + } + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index fce0d850..0ff1d017 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -57,7 +57,7 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId", "admittedMessageIdDuplicates"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) @@ -94,6 +94,18 @@ class WriteReceiptVectorTests(unittest.TestCase): failures.append(f"{case['name']}: expected {case['expect']}") self.assertEqual(failures, []) + def test_a_repeated_message_id_key_follows_the_last_key_wins_policy(self) -> None: + # Python decodes the frame to a dict, so it cannot see a duplicate: it judges the last key (admittedMessageIdDuplicates). + section = _VECTORS["admittedMessageIdDuplicates"] + self.assertEqual(_VECTORS["counts"]["admittedMessageIdDuplicateCases"], len(section["cases"])) + for case in section["cases"]: + raw = bytes.fromhex(case["wireHex"]) + if case["lastKeyWins"] == "refuse": + with self.assertRaises(ValueError, msg=case["name"]): + parse_message(raw) + else: + self.assertEqual(parse_message(raw).body["messageId"], case["lastKeyWinsEcho"], case["name"]) + def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None: # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 10d7dcb8..ee05bd72 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -238,3 +238,17 @@ fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { } assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } + +/// Vectors `admittedMessageIdDuplicates`: Rust decodes a map it can see into, so a repeated messageId key is refused whatever the values. +#[test] +fn a_repeated_message_id_key_is_refused_whatever_the_values() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + let vectors: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = vectors["admittedMessageIdDuplicates"]["cases"].as_array().unwrap(); + assert_eq!(vectors["counts"]["admittedMessageIdDuplicateCases"].as_u64().unwrap() as usize, cases.len()); + for case in cases { + assert_eq!(case["seesDuplicate"], "refuse"); + let decoded = decode_cultnet_message_from_slice(&unhex(case["wireHex"].as_str().unwrap()), CultNetWireContract::CultNetSchemaV0); + assert!(decoded.is_err(), "{}: a repeated messageId key must be refused", case["name"]); + } +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index c654de1a..1483fe31 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -323,3 +323,15 @@ test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned } assert.deepEqual(failures, []); }); + +// TypeScript decodes the frame to an object, so it cannot see a duplicate key: it judges the last one (admittedMessageIdDuplicates). +test("write_admitted messageId: a repeated key follows the last-key-wins policy", () => { + const vectors = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); + const section = vectors.admittedMessageIdDuplicates; + assert.equal(vectors.counts.admittedMessageIdDuplicateCases, section.cases.length); + for (const vector of section.cases as { name: string; wireHex: string; lastKeyWins: "accept" | "refuse"; lastKeyWinsEcho?: string }[]) { + const frame = decode(Uint8Array.from(Buffer.from(vector.wireHex, "hex"))); + if (vector.lastKeyWins === "refuse") assert.throws(() => parseCultNetMessage(frame), /./s, vector.name); + else assert.equal((parseCultNetMessage(frame) as CultNetWriteAdmittedMessage).messageId, vector.lastKeyWinsEcho, vector.name); + } +}); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index a4f1475c..6c65734c 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -87,5 +87,19 @@ namespace GameCult.Networking.Tests if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}"); Assert.That(failures, Is.Empty); } + + // admittedMessageIdDuplicates: C# reads the raw map, so it sees a repeated messageId key and refuses it whatever the values. + [Test] + public void ARepeatedMessageIdKeyIsRefusedWhateverTheValues() + { + var cases = Vectors().GetProperty("admittedMessageIdDuplicates").GetProperty("cases").EnumerateArray().ToArray(); + Assert.That(cases.Length, Is.EqualTo(Vectors().GetProperty("counts").GetProperty("admittedMessageIdDuplicateCases").GetInt32())); + foreach (var vector in cases) + { + Assert.That(vector.GetProperty("seesDuplicate").GetString(), Is.EqualTo("refuse")); + Assert.That(() => CultNetSchemaMessageSerialization.Deserialize(Convert.FromHexString(vector.GetProperty("wireHex").GetString()!)), + Throws.Exception, vector.GetProperty("name").GetString()); + } + } } } From e08d5c47f4cfe45e978c2ac5b8399b4cdd63b6b3 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:05:21 +0000 Subject: [PATCH 115/117] GameCult.Networking: a write_admitted frame with more than one messageId key is malformed Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index cb33b707..1e7bdd08 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -111,21 +111,24 @@ namespace GameCult.Networking { private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); - /// Throws unless every top-level messageId key of the frame is a valid write_admitted id; other keys are left to the serializer. + /// Throws unless the frame's one top-level messageId key is a valid write_admitted id (a repeated key is malformed); other keys are left to the serializer. public static void Require(byte[] payload) { var reader = new MessagePackReader(payload); var entries = reader.ReadMapHeader(); + var seen = false; for (var index = 0; index < entries; index++) { var key = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; if (key == "messageId" && reader.NextMessagePackType == MessagePackType.String) { + if (seen) throw new MessagePackSerializationException("cultnet.write_admitted.v0 carries messageId more than once."); + seen = true; string id; try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); - continue; // a repeated messageId key is checked too: the serializer keeps the last, so every one must be valid + continue; } if (key == null) reader.Skip(); From 27d9fbca9414cdaadb214095f66b9f643854f95a Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:07:11 +0000 Subject: [PATCH 116/117] write-receipt.md: where a received messageId decides, and the single-parser assumption it relies on Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 4f6cb7d1..5bb0a3e8 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,9 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. Every `messageId` key of a frame must be valid. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. A frame with more than one `messageId` key is malformed: a decoder that can see the duplicate (the C# and Rust decoders) refuses it; a decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses if that value is invalid, so no decoder echoes an invalid id (vectors `admittedMessageIdDuplicates` declare each policy and every runner asserts its own). A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. + +**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on every raw schema frame, write frames included; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. From 2aade7fdd32fd4ca5719c45d76365d545d546eba Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:25:50 +0000 Subject: [PATCH 117/117] Docs and test comments: correct the Rust duplicate-key citation, say what the TS and Rust duplicate tests pin Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- packages/cultnet-rs/tests/write_receipt.rs | 3 ++- packages/cultnet-ts/test/write-receipt.test.ts | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 5bb0a3e8..bfcbbb86 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -14,7 +14,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. A frame with more than one `messageId` key is malformed: a decoder that can see the duplicate (the C# and Rust decoders) refuses it; a decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses if that value is invalid, so no decoder echoes an invalid id (vectors `admittedMessageIdDuplicates` declare each policy and every runner asserts its own). A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. -**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on every raw schema frame, write frames included; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. +**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on write frames and on `write_admitted` (serde's `duplicate field messageId`, and `refuse_duplicate_keys` for the raw-list frames such as `document_put_raw`), though its `readable_message_id` names the first key when it names an unreadable frame; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index ee05bd72..3dd02a76 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -239,7 +239,8 @@ fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } -/// Vectors `admittedMessageIdDuplicates`: Rust decodes a map it can see into, so a repeated messageId key is refused whatever the values. +/// Vectors `admittedMessageIdDuplicates`: a repeated messageId key is refused whatever the values. This pins the outcome, not the +/// cause (today serde's `duplicate field messageId` in the typed decode of write_admitted). #[test] fn a_repeated_message_id_key_is_refused_whatever_the_values() { let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 1483fe31..23ef478e 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -325,6 +325,8 @@ test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned }); // TypeScript decodes the frame to an object, so it cannot see a duplicate key: it judges the last one (admittedMessageIdDuplicates). +// This pins @msgpack/msgpack's last-key behaviour plus the id check on the decoded object, not a first-key or duplicate path in +// cultnet-ts source: there is none, so a refuse-on-first mutation in src would not be killed. test("write_admitted messageId: a repeated key follows the last-key-wins policy", () => { const vectors = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); const section = vectors.admittedMessageIdDuplicates;