From f6c61b1ebf2aa7723362d30cf565ab9f188952f0 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:08 +0000 Subject: [PATCH 01/40] cultnet-py: decode cultnet.write_admitted.v0 and cultnet.error.v1 against the write-receipt vectors Strict closed-schema decode, content hash helper, hello capability; vectors contentHash, wire and schemaInvalid replayed. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-py/CHANGELOG.md | 1 + .../cultnet-py/src/cultnet_py/__init__.py | 8 + .../cultnet-py/src/cultnet_py/messages.py | 90 +++++-- .../cultnet-py/tests/test_write_receipt.py | 241 ++++++++++++++++++ 4 files changed, 317 insertions(+), 23 deletions(-) create mode 100644 packages/cultnet-py/tests/test_write_receipt.py diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index b6bfa92b..5e4b0cfa 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client. - Added `message_invalid` to `CULTNET_ERROR_CODES` and `message_invalid_error(field, reason)`, byte-identical to the C# `ForMessageInvalid`. `cultmesh-py` refuses a malformed put or delete with it instead of the out-of-schema `malformed_document_put` and `malformed_document_delete`, which are removed. - Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per remote endpoint, ported from the C# `CultNetRudpSocketTransportServer` and held to its bytes by diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index bcf2fcf0..e4a22779 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -39,6 +39,8 @@ from .cultmesh_contracts import ( from .framing import decode_frame, encode_frame, read_frame, write_frame from .messages import ( CULTNET_ERROR_CODES, + CULTNET_ERROR_V1_CODES, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, CultNetMessage, CultNetWitnessArtifactBundle, compute_simulation_claim_hash, @@ -51,7 +53,9 @@ from .messages import ( error_message, message_invalid_error, unsupported_schema_version_error, + write_content_hash, hello, + hello_advertises_write_receipts, login, login_success, parse_message, @@ -176,6 +180,8 @@ __all__ = [ "verse_catalog_request", "verse_from_wire", "CULTNET_ERROR_CODES", + "CULTNET_ERROR_V1_CODES", + "CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS", "CultNetMessage", "CultNetWitnessArtifactBundle", "CultNetDatabaseSubscription", @@ -266,6 +272,8 @@ __all__ = [ "message_invalid_error", "unsupported_schema_version_error", "hello", + "hello_advertises_write_receipts", + "write_content_hash", "INTEROP_WIRE_CONTRACT", "login", "login_success", diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index 1a1f8e88..f2d89296 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -29,7 +29,11 @@ def parse_message(payload: bytes) -> CultNetMessage: body = dict(decoded) del body["schemaVersion"] if schema_version == "cultnet.error.v0": - _validate_error_body(body) + _validate_error_body(body, "cultnet.error.v0", CULTNET_ERROR_CODES) + elif schema_version == "cultnet.error.v1": + _validate_error_body(body, "cultnet.error.v1", CULTNET_ERROR_V1_CODES) + elif schema_version == "cultnet.write_admitted.v0": + _validate_write_admitted_body(body) return CultNetMessage(schema_version=schema_version, body=body) @@ -47,6 +51,13 @@ CULTNET_ERROR_CODES: tuple[str, ...] = ( "message_invalid", ) +# cultnet.error.v1 (contracts/cultnet/interop/cultnet.error.v1.schema.json): v0's codes plus limit_reached. +CULTNET_ERROR_V1_CODES: tuple[str, ...] = (*CULTNET_ERROR_CODES, "limit_reached") + +# The two frames a receipt-capable hello lists in supportedMessageVersions; a peer lists both or neither +# (contracts/cultnet/interop/write-receipt.md, "Capability: opting in"). +CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS: tuple[str, ...] = ("cultnet.write_admitted.v0", "cultnet.error.v1") + _ERROR_DETAILS_KEYS = ("field", "value", "asOf", "current") @@ -54,16 +65,48 @@ def _is_uint(value: Any) -> bool: return isinstance(value, int) and not isinstance(value, bool) and value >= 0 -def _validate_error_body(body: dict[str, Any]) -> None: - """Refuse a cultnet.error.v0 body that is not the full-key shape the schema requires.""" - expected = ("error", "routingHint", "code", "details") +def _is_name(value: Any) -> bool: + return isinstance(value, str) and value != "" + + +def _validate_write_admitted_body(body: dict[str, Any]) -> None: + """Refuse a cultnet.write_admitted.v0 body that is not exactly the five keys of its closed schema.""" + expected = ("messageId", "operation", "schemaId", "recordKey") if set(body) != set(expected): - raise ValueError(f"cultnet.error.v0 must carry exactly {list(expected)}, got {sorted(body)}") + raise ValueError(f"cultnet.write_admitted.v0 must carry exactly {list(expected)}, got {sorted(body)}") + for key in ("messageId", "schemaId", "recordKey"): + if not _is_name(body[key]): + raise ValueError(f"cultnet.write_admitted.v0 {key} must be a non-empty string") + if body["operation"] not in ("put", "delete"): + raise ValueError("cultnet.write_admitted.v0 operation must be put or delete") + + +def write_content_hash(operation: str, schema_id: str, record_key: str, payload: bytes | None) -> bytes: + """SHA-256 of the MessagePack array [operation, schemaId, recordKey, payload] (write-receipt.md, rule 5). + + payload is the raw document payload, packed as a MessagePack bin for a put and nil for a delete. + """ + return hashlib.sha256(msgpack.packb([operation, schema_id, record_key, payload], use_bin_type=True)).digest() + + +def hello_advertises_write_receipts(hello_message: CultNetMessage) -> bool: + """True when a hello lists both receipt frames: the only thing that makes a connection receipt-capable.""" + listed = hello_message.body.get("supportedMessageVersions") + return isinstance(listed, list) and all(version in listed for version in CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS) + + +def _validate_error_body(body: dict[str, Any], version: str, codes: tuple[str, ...]) -> None: + """Refuse a cultnet.error body that is not the full-key shape its closed schema requires.""" + expected = ("error", "routingHint", "code", "details") + (("messageId",) if version == "cultnet.error.v1" else ()) + if set(body) != set(expected): + raise ValueError(f"{version} must carry exactly {list(expected)}, got {sorted(body)}") if not isinstance(body["error"], str) or not body["error"]: - raise ValueError("cultnet.error.v0 error must be a non-empty string") + raise ValueError(f"{version} error must be a non-empty string") code = body["code"] - if code is not None and (not isinstance(code, str) or code not in CULTNET_ERROR_CODES): - raise ValueError(f"cultnet.error.v0 code {code!r} is not a recognized refusal code") + if code is not None and (not isinstance(code, str) or code not in codes): + raise ValueError(f"{version} code {code!r} is not a recognized refusal code") + if "messageId" in body and body["messageId"] is not None and not _is_name(body["messageId"]): + raise ValueError(f"{version} messageId must be a non-empty string or null") hint = body["routingHint"] if hint is not None: if ( @@ -72,17 +115,17 @@ def _validate_error_body(body: dict[str, Any]) -> None: or not isinstance(hint["reason"], str) or not (hint["shard"] is None or isinstance(hint["shard"], dict)) ): - raise ValueError("cultnet.error.v0 routingHint must be null or {shard, reason}") + raise ValueError(f"{version} routingHint must be null or {{shard, reason}}") details = body["details"] if details is not None: if not isinstance(details, dict) or set(details) != set(_ERROR_DETAILS_KEYS): - raise ValueError(f"cultnet.error.v0 details must be null or carry exactly {list(_ERROR_DETAILS_KEYS)}") + raise ValueError(f"{version} details must be null or carry exactly {list(_ERROR_DETAILS_KEYS)}") for key in ("field", "value"): if details[key] is not None and not isinstance(details[key], str): - raise ValueError(f"cultnet.error.v0 details.{key} must be a string or null") + raise ValueError(f"{version} details.{key} must be a string or null") for key in ("asOf", "current"): if details[key] is not None and not _is_uint(details[key]): - raise ValueError(f"cultnet.error.v0 details.{key} must be a non-negative integer or null") + raise ValueError(f"{version} details.{key} must be a non-negative integer or null") def hello( @@ -92,17 +135,18 @@ def hello( supported_schema_versions: list[str] | None = None, supported_mutation_contracts: list[dict[str, Any]] | None = None, transport_profiles: list[dict[str, Any]] | None = None, + supported_message_versions: list[str] | None = None, ) -> CultNetMessage: - return CultNetMessage( - "cultnet.hello.v0", - { - "runtimeId": runtime_id, - "supportedWireContracts": supported_wire_contracts or ["cultnet.schema.v0"], - "supportedSchemaVersions": supported_schema_versions or [], - "supportedMutationContracts": supported_mutation_contracts or [], - "transportProfiles": transport_profiles or [], - }, - ) + body: dict[str, Any] = { + "runtimeId": runtime_id, + "supportedWireContracts": supported_wire_contracts or ["cultnet.schema.v0"], + "supportedSchemaVersions": supported_schema_versions or [], + "supportedMutationContracts": supported_mutation_contracts or [], + "transportProfiles": transport_profiles or [], + } + if supported_message_versions is not None: + body["supportedMessageVersions"] = supported_message_versions + return CultNetMessage("cultnet.hello.v0", body) def login( @@ -172,7 +216,7 @@ def error_message( } if details is not None and set(details) - set(_ERROR_DETAILS_KEYS): raise ValueError(f"cultnet.error.v0 details has unknown keys {sorted(set(details) - set(_ERROR_DETAILS_KEYS))}") - _validate_error_body(body) + _validate_error_body(body, "cultnet.error.v0", CULTNET_ERROR_CODES) return CultNetMessage("cultnet.error.v0", body) diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py new file mode 100644 index 00000000..32c5f03a --- /dev/null +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import json +import unittest +from pathlib import Path +from typing import Any + +import msgpack + +from cultnet_py import ( + CULTNET_ERROR_CODES, + CULTNET_ERROR_V1_CODES, + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + error_message, + hello, + hello_advertises_write_receipts, + parse_message, + write_content_hash, +) + +# Decoder-side conformance with contracts/cultnet/interop/write-receipt-vectors.json: `contentHash`, `wire` and +# `schemaInvalid`. `dedupe` and `blankFields` are server obligations and `scenarios` is the client write tracker; +# this runtime has neither a receipt server nor a write client, so it replays none of them. +_INTEROP = Path(__file__).resolve().parents[3] / "contracts/cultnet/interop" +_VECTORS = json.loads((_INTEROP / "write-receipt-vectors.json").read_text(encoding="utf-8")) +_ERROR_V1_SCHEMA = json.loads((_INTEROP / "cultnet.error.v1.schema.json").read_text(encoding="utf-8")) + + +def _pack(wire: dict[str, Any]) -> bytes: + return msgpack.packb(wire, use_bin_type=True) + + +def _admitted(**override: Any) -> dict[str, Any]: + return { + "schemaVersion": "cultnet.write_admitted.v0", + "messageId": "put-1", + "operation": "put", + "schemaId": "note.v0", + "recordKey": "note:1", + **override, + } + + +def _refused(**override: Any) -> dict[str, Any]: + return { + "schemaVersion": "cultnet.error.v1", + "error": "write_not_authorized: no", + "routingHint": None, + "code": "write_not_authorized", + "details": None, + "messageId": "put-1", + **override, + } + + +class WriteReceiptVectorTests(unittest.TestCase): + def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: + self.assertEqual( + set(_VECTORS) - {"provenance", "specification"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios"}, + ) + self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) + self.assertEqual(len(_VECTORS["wire"]), 7) + self.assertEqual(len(_VECTORS["schemaInvalid"]), 7) + + def test_content_hash_matches_every_vector(self) -> None: + for case in _VECTORS["contentHash"]["cases"]: + payload = None if case["payloadHex"] is None else bytes.fromhex(case["payloadHex"]) + digest = write_content_hash(case["operation"], case["schemaId"], case["recordKey"], payload) + self.assertEqual(digest.hex(), case["sha256"], case["name"]) + + def test_wire_vectors_decode_to_their_fields_in_order_and_reencode_byte_identical(self) -> None: + for vector in _VECTORS["wire"]: + raw = bytes.fromhex(vector["messagePackHex"]) + message = parse_message(raw) + self.assertEqual(message.to_wire(), vector["message"], vector["name"]) + self.assertEqual(list(message.to_wire()), list(vector["message"]), vector["name"]) + self.assertEqual(message.to_bytes(), raw, vector["name"]) + + def test_schema_invalid_vectors_are_refused(self) -> None: + for vector in _VECTORS["schemaInvalid"]: + with self.assertRaises(ValueError, msg=vector["name"]): + parse_message(bytes.fromhex(vector["messagePackHex"])) + + def test_a_payload_that_is_not_bytes_changes_the_hash(self) -> None: + # The payload is a bin, not an str and not a re-parsed document. + as_bin = write_content_hash("put", "note.v0", "note:1", b"\x81\xa2id\xa11") + self.assertNotEqual(as_bin, write_content_hash("put", "note.v0", "note:1", None)) + self.assertEqual(len(as_bin), 32) + + +class ErrorV1Tests(unittest.TestCase): + def test_code_set_equals_the_schema_enum_and_v0_is_unchanged(self) -> None: + schema_codes = [c for c in _ERROR_V1_SCHEMA["properties"]["code"]["enum"] if c is not None] + self.assertEqual(sorted(CULTNET_ERROR_V1_CODES), sorted(schema_codes)) + self.assertNotIn("limit_reached", CULTNET_ERROR_CODES) + self.assertEqual(CULTNET_ERROR_V1_CODES[-1], "limit_reached") + + def test_every_v1_code_decodes_and_an_unknown_one_does_not(self) -> None: + for code in CULTNET_ERROR_V1_CODES: + self.assertEqual(parse_message(_pack(_refused(code=code))).body["code"], code) + parse_message(_pack(_refused(code=None))) + with self.assertRaises(ValueError): + parse_message(_pack(_refused(code="invented"))) + with self.assertRaises(ValueError): + parse_message(_pack(_refused(code=7))) + + def test_limit_reached_is_v1_only(self) -> None: + v0 = {k: v for k, v in _refused(code="limit_reached", schemaVersion="cultnet.error.v0").items() if k != "messageId"} + with self.assertRaises(ValueError): + parse_message(_pack(v0)) + with self.assertRaises(ValueError): + error_message("x", code="limit_reached") + + def test_message_id_is_required_and_is_a_nonempty_string_or_null(self) -> None: + self.assertEqual(parse_message(_pack(_refused(messageId="w-é 1"))).body["messageId"], "w-é 1") + self.assertIsNone(parse_message(_pack(_refused(messageId=None))).body["messageId"]) + omitted = {k: v for k, v in _refused().items() if k != "messageId"} + with self.assertRaises(ValueError): + parse_message(_pack(omitted)) + for bad in ("", 5, b"put-1", ["put-1"], True): + with self.assertRaises(ValueError, msg=repr(bad)): + parse_message(_pack(_refused(messageId=bad))) + + def test_v0_stays_closed_to_message_id(self) -> None: + v0 = _refused(schemaVersion="cultnet.error.v0", code="message_invalid") + with self.assertRaises(ValueError): + parse_message(_pack(v0)) + del v0["messageId"] + self.assertEqual(parse_message(_pack(v0)).schema_version, "cultnet.error.v0") + + def test_v1_is_closed_to_unknown_keys_and_keeps_the_shape_rules_of_v0(self) -> None: + for bad in ( + _refused(extra=1), + _refused(error=""), + _refused(error=3), + _refused(routingHint={"shard": None}), + _refused(routingHint={"shard": 1, "reason": "r"}), + _refused(routingHint={"shard": None, "reason": 1}), + _refused(details={"field": None, "value": None, "asOf": None}), + _refused(details={"field": 1, "value": None, "asOf": None, "current": None}), + _refused(details={"field": None, "value": None, "asOf": -1, "current": None}), + _refused(details={"field": None, "value": None, "asOf": None, "current": True}), + ): + with self.assertRaises(ValueError, msg=str(bad)): + parse_message(_pack(bad)) + for good in ( + _refused(routingHint={"shard": None, "reason": "r"}), + _refused(details={"field": "messageId", "value": "x", "asOf": 1, "current": 2}), + ): + parse_message(_pack(good)) + + def test_the_refusal_reason_names_its_version(self) -> None: + with self.assertRaisesRegex(ValueError, "cultnet.error.v1 code"): + parse_message(_pack(_refused(code="invented"))) + with self.assertRaisesRegex(ValueError, r"routingHint must be null or \{shard, reason\}"): + parse_message(_pack(_refused(routingHint={"shard": None}))) + + +class RefusalTextTests(unittest.TestCase): + """A refusal names its frame version and the rule it broke; a caller reads this text in a log.""" + + def assertRefused(self, wire: dict[str, Any], pattern: str) -> None: + with self.assertRaisesRegex(ValueError, pattern): + parse_message(_pack(wire)) + + def test_error_frames_name_their_own_version(self) -> None: + v0 = {k: v for k, v in _refused(schemaVersion="cultnet.error.v0").items() if k != "messageId"} + self.assertRefused({**v0, "code": "nope"}, r"^cultnet\.error\.v0 code") + self.assertRefused({**v0, "extra": 1}, r"^cultnet\.error\.v0 must carry exactly") + self.assertRefused(_refused(extra=1), r"^cultnet\.error\.v1 must carry exactly") + self.assertRefused(_refused(error=""), r"^cultnet\.error\.v1 error must be") + self.assertRefused(_refused(messageId=""), r"^cultnet\.error\.v1 messageId must be") + self.assertRefused(_refused(details={"field": None}), r"^cultnet\.error\.v1 details must be null or carry exactly") + self.assertRefused( + _refused(details={"field": 1, "value": None, "asOf": None, "current": None}), r"^cultnet\.error\.v1 details\.field must be" + ) + self.assertRefused( + _refused(details={"field": None, "value": None, "asOf": -1, "current": None}), r"^cultnet\.error\.v1 details\.asOf must be" + ) + with self.assertRaisesRegex(ValueError, r"^cultnet\.error\.v0 code"): + error_message("x", code="bogus") + + def test_write_admitted_names_the_broken_rule(self) -> None: + self.assertRefused(_admitted(extra=1), r"^cultnet\.write_admitted\.v0 must carry exactly") + self.assertRefused(_admitted(recordKey=""), r"^cultnet\.write_admitted\.v0 recordKey must be a non-empty string") + self.assertRefused(_admitted(operation="upsert"), r"^cultnet\.write_admitted\.v0 operation must be put or delete$") + + +class WriteAdmittedTests(unittest.TestCase): + def test_both_operations_decode(self) -> None: + for operation in ("put", "delete"): + self.assertEqual(parse_message(_pack(_admitted(operation=operation))).body["operation"], operation) + + def test_every_key_is_required_and_no_other_is_allowed(self) -> None: + for key in ("messageId", "operation", "schemaId", "recordKey"): + with self.assertRaises(ValueError, msg=key): + parse_message(_pack({k: v for k, v in _admitted().items() if k != key})) + for extra in ("error", "code", "details"): + with self.assertRaises(ValueError, msg=extra): + parse_message(_pack(_admitted(**{extra: "x"}))) + + def test_each_string_is_nonempty_and_a_string(self) -> None: + for key in ("messageId", "schemaId", "recordKey"): + for bad in ("", None, 3, b"x"): + with self.assertRaises(ValueError, msg=f"{key}={bad!r}"): + parse_message(_pack(_admitted(**{key: bad}))) + + def test_operation_is_put_or_delete(self) -> None: + for bad in ("upsert", "PUT", "", None, 1): + with self.assertRaises(ValueError, msg=repr(bad)): + parse_message(_pack(_admitted(operation=bad))) + + def test_message_id_is_echoed_byte_for_byte(self) -> None: + self.assertEqual(parse_message(_pack(_admitted(messageId=" w é 1 "))).body["messageId"], " w é 1 ") + + +class HelloCapabilityTests(unittest.TestCase): + def test_capability_needs_both_frames(self) -> None: + both = list(CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS) + self.assertEqual(both, ["cultnet.write_admitted.v0", "cultnet.error.v1"]) + self.assertTrue(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=both))) + self.assertTrue(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=[*both, "x"]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=both[:1]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=both[1:]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r", supported_message_versions=[]))) + self.assertFalse(hello_advertises_write_receipts(hello(runtime_id="r"))) + + def test_a_non_list_listing_is_not_a_capability(self) -> None: + wire = hello(runtime_id="r").to_wire() + wire["supportedMessageVersions"] = "cultnet.write_admitted.v0 cultnet.error.v1" + self.assertFalse(hello_advertises_write_receipts(parse_message(_pack(wire)))) + + def test_hello_writes_the_listing_only_when_given_and_round_trips(self) -> None: + self.assertNotIn("supportedMessageVersions", hello(runtime_id="r").body) + listed = hello(runtime_id="r", supported_message_versions=list(CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS)) + self.assertEqual(parse_message(listed.to_bytes()), listed) + + +if __name__ == "__main__": + unittest.main() From 56b0c1679c4bc07940dcbcd69d773edb6fa27ecc Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:55 +0000 Subject: [PATCH 02/40] cultmesh-rs: receipt_interop_server example, the real RUDP document server on an ephemeral port for the interop harness Co-Authored-By: Claude Sonnet 5.5 --- .../examples/receipt_interop_server.rs | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 packages/cultmesh-rs/examples/receipt_interop_server.rs diff --git a/packages/cultmesh-rs/examples/receipt_interop_server.rs b/packages/cultmesh-rs/examples/receipt_interop_server.rs new file mode 100644 index 00000000..b8c66fd9 --- /dev/null +++ b/packages/cultmesh-rs/examples/receipt_interop_server.rs @@ -0,0 +1,77 @@ +//! The real Rust RUDP document server on an ephemeral loopback port, for the cross-runtime write-receipt interop +//! harness (packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts). It is the production server with a +//! recording sink and nothing else: stdout carries one JSON line per fact the harness asserts on, and the server runs +//! until stdin closes. +//! +//! {"event":"listening","port":N} +//! {"event":"applied","op":"put","messageId":"...","schemaId":"...","recordKey":"..."} +//! +//! Record keys that start with `refuse:` are refused by the sink (authorize and commit), which is how the harness +//! asks for a refused write. +use anyhow::Result; +use cultmesh_rs::{ + CultMeshRudpDocumentServer, CultMeshRudpRawDocumentReceipt, CultMeshRudpRawDocumentSink, + CultMeshRudpSnapshotQuery, CultMeshSystemClock, +}; +use cultnet_rs::CultNetRawDocumentRecord; +use serde_json::json; +use std::io::Read; +use std::net::UdpSocket; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; +use std::time::Duration; + +const REFUSED_PREFIX: &str = "refuse:"; + +struct RecordingSink; + +impl CultMeshRudpRawDocumentSink for RecordingSink { + fn authorize_raw_document(&mut self, receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + if receipt.document.record_key.starts_with(REFUSED_PREFIX) { + anyhow::bail!("not authorized"); + } + Ok(()) + } + + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + if receipt.document.record_key.starts_with(REFUSED_PREFIX) { + anyhow::bail!("not authorized"); + } + println!( + "{}", + json!({ + "event": "applied", + "op": "put", + "messageId": receipt.message_id, + "schemaId": receipt.document.schema_id, + "recordKey": receipt.document.record_key, + }) + ); + Ok(()) + } +} + +fn main() -> Result<()> { + let source = |_: &CultMeshRudpSnapshotQuery| -> Result> { Ok(Vec::new()) }; + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + RecordingSink, + source, + CultMeshSystemClock::default(), + Default::default(), + )?; + println!("{}", json!({ "event": "listening", "port": server.local_addr()?.port() })); + let stop = Arc::new(AtomicBool::new(false)); + let watcher = Arc::clone(&stop); + thread::spawn(move || { + let mut sink = Vec::new(); + let _ = std::io::stdin().read_to_end(&mut sink); + watcher.store(true, Ordering::SeqCst); + }); + while !stop.load(Ordering::SeqCst) { + server.poll_once()?; + thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} From e4d2446dbe4dd268ee3b7bd20f686d72e66a53e3 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:55 +0000 Subject: [PATCH 03/40] C# InteropPeer: receipt-serve (CultNetDatabaseServer on an ephemeral port) and receipt-relay (LiteNetLib byte shim for the TS client) Co-Authored-By: Claude Sonnet 5.5 --- .../Program.cs | 8 +- .../ReceiptHarness.cs | 118 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs diff --git a/tests/GameCult.Networking.InteropPeer/Program.cs b/tests/GameCult.Networking.InteropPeer/Program.cs index 17a1ea89..fcfcf09c 100644 --- a/tests/GameCult.Networking.InteropPeer/Program.cs +++ b/tests/GameCult.Networking.InteropPeer/Program.cs @@ -25,7 +25,7 @@ static async Task ProgramMainAsync(string[] args) { if (args.Length == 0) { - throw new InvalidOperationException("Expected mode: serve | probe | dial | rudp-serve-once | rudp-dial-once | rudp-serve-message-once | rudp-dial-message-once | quic-realtime-serve"); + throw new InvalidOperationException("Expected mode: serve | probe | dial | rudp-serve-once | rudp-dial-once | rudp-serve-message-once | rudp-dial-message-once | quic-realtime-serve | receipt-serve | receipt-relay"); } var mode = args[0]; @@ -53,6 +53,12 @@ static async Task ProgramMainAsync(string[] args) case "rudp-dial-message-once": RudpDialMessageOnce(options); return 0; + case "receipt-serve": + await GameCult.Networking.InteropPeer.ReceiptHarness.ServeAsync(); + return 0; + case "receipt-relay": + await GameCult.Networking.InteropPeer.ReceiptHarness.RelayAsync(int.Parse(options["port"], CultureInfo.InvariantCulture)); + return 0; case "quic-realtime-serve": await QuicRealtimeServeAsync(ParseQuicRealtimeServeConfig(options)); return 0; diff --git a/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs new file mode 100644 index 00000000..75eaf4da --- /dev/null +++ b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs @@ -0,0 +1,118 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using LiteNetLib; +using R3; + +// The real C# receipt server (CultNetDatabaseServer on Server) and the one transport shim the TypeScript client needs to reach it, +// for the cross-runtime write-receipt interop harness (packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts). +// TypeScript has no LiteNetLib client, so the shim moves bytes and nothing else: each CultNet frame is a LiteNetLib reliable +// ordered payload, and on the pipe it is a 4-byte big-endian length followed by the frame. +namespace GameCult.Networking.InteropPeer +{ + [CultDocument("receipt.interop_note", ReceiptHarness.SchemaId)] + [MessagePack.MessagePackObject] + public sealed class ReceiptInteropNote + { + [MessagePack.Key(0)] + public string Text { get; set; } = string.Empty; + } + + internal static class ReceiptHarness + { + public const string SchemaId = "receipt.interop_note.v1"; + public const string ShardId = "receipt-interop-shard"; + // A record key with this prefix is refused by the authority: how the harness asks for a refused write. + public const string RefusedPrefix = "refuse:"; + + private sealed class Authority : ICultNetRemoteWriteAuthority + { + public CultNetRemoteWriteDecision Decide(CultNetRemoteWriteRequest request) => + request.RecordKey.ToString().StartsWith(RefusedPrefix, StringComparison.Ordinal) + ? CultNetRemoteWriteDecision.Refuse("denied") + : CultNetRemoteWriteDecision.Grant(); + } + + private static void Emit(object fact) + { + lock (Console.Out) Console.Out.WriteLine(JsonSerializer.Serialize(fact)); + } + + // stdout: {"event":"listening","port":N} then {"event":"applied"|"removed","recordKey":"..."} per committed change; runs until stdin closes. + public static async Task ServeAsync() + { + var cache = new CultCache(); + var database = new CultNetDatabase(cache, new CultNetDatabaseOptions + { + Shards = [new CultNetShardDescriptor(ShardId, "primary", epoch: 1, isPrimary: true, schemaIds: [SchemaId])] + }); + using var server = new Server(cache, ServerSecurityOptions.Development()) { Port = 0 }; + using var databaseServer = new CultNetDatabaseServer(server, database, new CultNetDatabaseServerOptions + { + RemoteWriteAuthority = new Authority() + }); + using var watch = cache.Watch().Subscribe(change => + Emit(new { @event = change.Kind == CultCacheDocumentChangeKind.Removed ? "removed" : "applied", recordKey = change.Key.ToString() })); + server.Start(); + Emit(new { @event = "listening", port = server.Port, shardId = ShardId, schemaId = SchemaId }); + await Task.Run(() => Console.In.ReadToEnd()); + } + + // Connects to the server on `port` and relays frames between stdin/stdout and the connection; exits when either ends. + public static async Task RelayAsync(int port) + { + var listener = new EventBasedNetListener(); + var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var ended = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var output = Console.OpenStandardOutput(); + var gate = new object(); + listener.PeerConnectedEvent += _ => connected.TrySetResult(true); + listener.PeerDisconnectedEvent += (_, _) => ended.TrySetResult(true); + listener.NetworkReceiveEvent += (_, reader, _, _) => + { + var frame = reader.GetRemainingBytes(); + var header = new byte[] { (byte)(frame.Length >> 24), (byte)(frame.Length >> 16), (byte)(frame.Length >> 8), (byte)frame.Length }; + lock (gate) + { + output.Write(header); + output.Write(frame); + output.Flush(); + } + }; + var manager = new NetManager(listener) { UnsyncedEvents = true }; + manager.Start(0); + var peer = manager.Connect("127.0.0.1", port, ServerSecurityOptions.Development().ConnectionKey); + if (await Task.WhenAny(connected.Task, Task.Delay(TimeSpan.FromSeconds(10))) != connected.Task) + throw new InvalidOperationException("the relay did not connect"); + var input = Console.OpenStandardInput(); + var pump = Task.Run(() => + { + var header = new byte[4]; + while (ReadExactly(input, header)) + { + var frame = new byte[(header[0] << 24) | (header[1] << 16) | (header[2] << 8) | header[3]]; + if (!ReadExactly(input, frame)) break; + peer.Send(frame, DeliveryMethod.ReliableOrdered); + } + }); + await Task.WhenAny(pump, ended.Task); + manager.Stop(); + } + + private static bool ReadExactly(Stream stream, byte[] buffer) + { + var read = 0; + while (read < buffer.Length) + { + var count = stream.Read(buffer, read, buffer.Length - read); + if (count <= 0) return false; + read += count; + } + return true; + } + } +} From 140eca51384ddac73c61dff68afa2bee5913881c Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:06:55 +0000 Subject: [PATCH 04/40] cultmesh-ts: TS write-receipt client against the real Rust and C# receipt servers (test:interop) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 + packages/cultmesh-ts/package.json | 3 +- .../interop/write-receipt-interop.test.ts | 418 ++++++++++++++++++ 3 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 35756ecf..4b13aa72 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -78,6 +78,8 @@ Those clients complete *successfully* when the window ends with no refusal. Unde Advertising is per server class, not per transport. A class advertises both frames in its hello only if it implements the receipt rules end to end; the C# RUDP schema server stays "no" until it answers writes with receipts. +Cross-runtime check: `packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts` (`npm run test:interop` in `packages/cultmesh-ts`) runs the TypeScript client (`WriteTracker`, the decoder, the one-shot writer) against the real Rust RUDP document server and the real `CultNetDatabaseServer` over loopback, one scenario set against both. Known gap it records as a todo: the Rust RUDP document server refuses every delete (`write_not_authorized: the server does not take deletes`), where this spec has an admitted delete receipted like a put. + ## Conformance follow-ups (not in this PR) C# server and client, Rust, TypeScript, Python, Kotlin: emit and parse the two frames; hello capability; tri-state client outcome; replace the #55 to #58 window-success; re-capture the wire vectors in each runtime. Server write paths include the shard write forwarder (rule 6) and the Odin catalog. diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index 92962cbc..0fef7419 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -24,7 +24,8 @@ "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", - "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --expose-gc --test dist-test/test/*.test.js" + "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --expose-gc --test dist-test/test/*.test.js", + "test:interop": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/write-receipt-interop.test.js" }, "keywords": [ "cultmesh", diff --git a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts new file mode 100644 index 00000000..8a646a50 --- /dev/null +++ b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts @@ -0,0 +1,418 @@ +// The TypeScript write-receipt client (cultnet-ts WriteTracker and decoder, cultmesh-ts one-shot writer) against the two real +// receipt servers over loopback, as child processes on ephemeral ports: +// +// rust packages/cultmesh-rs/examples/receipt_interop_server.rs, the RUDP document server, reached over RUDP. +// csharp tests/GameCult.Networking.InteropPeer `receipt-serve`, CultNetDatabaseServer on Server (LiteNetLib). TypeScript has no +// LiteNetLib client, so a second child, `receipt-relay`, carries bytes between the test and the server (4-byte +// big-endian length, then the CultNet frame: CultNetPeer's own stream framing). It decodes and judges nothing. +// +// Every scenario runs against both targets; its outcome is read from the WriteTracker (admitted, refused, unconfirmed), and the +// server's own stdout facts say what was committed. Contract: contracts/cultnet/interop/write-receipt.md and its vectors. +// +// Needs cargo and dotnet; run it with `npm run test:interop` in packages/cultmesh-ts (CARGO and DOTNET override the commands; +// RECEIPT_INTEROP_PREBUILT=1 skips the two builds). +import assert from "node:assert/strict"; +import { execFile, spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { Duplex } from "node:stream"; +import { after, before, describe, test } from "node:test"; +import { promisify } from "node:util"; +import { encode } from "@msgpack/msgpack"; +import { z } from "zod"; +import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS, + CultNetPeer, + WriteTracker, + defineCultNetDocumentBinding, + helloAdvertisesWriteReceipts, + resultWindowMs, + type CultNetErrorV1Message, + type CultNetHelloMessage, + type CultNetMessage, + type WriteSettlement, +} from "@gamecult/cultnet-ts"; +import { CultMesh, CultMeshRudpWriteRefusedError, CultMeshRudpWriteUnconfirmedError } from "../../src/index"; + +const execFileAsync = promisify(execFile); +// dist-test/test/interop -> the repository root +const repoRoot = resolve(__dirname, "..", "..", "..", "..", ".."); +const cargo = process.env.CARGO ?? "cargo"; +const dotnet = process.env.DOTNET ?? "dotnet"; +const rustManifestDir = join(repoRoot, "packages", "cultmesh-rs"); +const csharpProject = join(repoRoot, "tests", "GameCult.Networking.InteropPeer", "GameCult.Networking.InteropPeer.csproj"); +const csharpDll = join(repoRoot, "bin", "GameCult.Networking.InteropPeer", "Debug", "net10.0", "GameCult.Networking.InteropPeer.dll"); +// Set when the server binaries were built beforehand (the builds themselves cannot start under a tight address-space cap). +const prebuilt = process.env.RECEIPT_INTEROP_PREBUILT === "1"; +const vectors = JSON.parse(readFileSync(join(repoRoot, "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); + +const SCHEMA_ID = "receipt.interop_note.v1"; +const UNREADABLE = "message_invalid: the message could not be read"; +const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms)); + +async function waitFor(predicate: () => boolean, what: string, ms = 5_000): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > ms) throw new Error(`timed out waiting for ${what}`); + await sleep(5); + } +} + +// ---- the servers ------------------------------------------------------------------------------------------------------- + +interface Fact { event: string; recordKey?: string; messageId?: string; [key: string]: unknown } + +/** A running server child and the JSON facts it prints on stdout. */ +class ServerProcess { + readonly facts: Fact[] = []; + stderr = ""; + port = 0; + constructor(readonly child: ChildProcess) { + let pending = ""; + child.stdout!.on("data", (chunk: Buffer) => { + pending += chunk.toString("utf8"); + for (let at = pending.indexOf("\n"); at >= 0; at = pending.indexOf("\n")) { + const line = pending.slice(0, at).trim(); + pending = pending.slice(at + 1); + if (line) this.facts.push(JSON.parse(line) as Fact); + } + }); + child.stderr!.on("data", (chunk: Buffer) => { this.stderr += chunk.toString("utf8"); }); + } + /** How many times `event` was printed for `recordKey`. */ + count(event: string, recordKey: string): number { + return this.facts.filter((fact) => fact.event === event && fact.recordKey === recordKey).length; + } + async stop(): Promise { + this.child.stdin!.end(); + await new Promise((done) => { + const killer = setTimeout(() => this.child.kill("SIGKILL"), 5_000); + this.child.once("exit", () => { clearTimeout(killer); done(); }); + if (this.child.exitCode !== null) done(); + }); + } +} + +async function launch(command: string, args: string[]): Promise { + const server = new ServerProcess(spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] })); + await waitFor(() => server.facts.some((fact) => fact.event === "listening") || server.child.exitCode !== null, "the server to listen", 60_000); + const listening = server.facts.find((fact) => fact.event === "listening"); + if (!listening) throw new Error(`the server exited before listening:\n${server.stderr}`); + server.port = listening.port as number; + return server; +} + +/** One connection to a server: the peer that speaks CultNet, and how to close it. */ +interface Connection { peer: CultNetPeer; close(): void } + +interface Target { + name: string; + /** Shard fields the server's database requires on a write, if any. */ + shard?: { shardId: string; shardEpoch: number }; + /** A document payload this target's store reads. */ + payload(text: string): Uint8Array; + start(): Promise; + connect(server: ServerProcess): Promise; +} + +let nextConnectionId = 0x20206000; + +const rust: Target = { + name: "rust", + payload: (text) => encode({ text }), + async start() { + if (!prebuilt) await execFileAsync(cargo, ["build", "--example", "receipt_interop_server", "--message-format", "short"], { cwd: rustManifestDir }); + const metadata = spawnSync(cargo, ["metadata", "--format-version", "1", "--no-deps"], { cwd: rustManifestDir, encoding: "utf8" }); + if (metadata.status !== 0) throw new Error(`cargo metadata failed: ${metadata.stderr}`); + const exe = join(JSON.parse(metadata.stdout).target_directory, "debug", "examples", `receipt_interop_server${process.platform === "win32" ? ".exe" : ""}`); + return launch(exe, []); + }, + async connect(server) { + const peer = await CultMesh.createRudpPeer("receipt-interop", nextConnectionId++, { host: "127.0.0.1", port: server.port, uri: `rudp://127.0.0.1:${server.port}` }); + return { peer, close: () => peer.close() }; + }, +}; + +const csharp: Target = { + name: "csharp", + shard: { shardId: "receipt-interop-shard", shardEpoch: 1 }, + payload: (text) => encode([text]), + async start() { + if (!prebuilt) await execFileAsync(dotnet, ["build", csharpProject, "-nologo", "-v", "q"], { cwd: repoRoot, maxBuffer: 64 * 1024 * 1024 }); + return launch(dotnet, [csharpDll, "receipt-serve"]); + }, + async connect(server) { + const relay = spawn(dotnet, [csharpDll, "receipt-relay", "--port", String(server.port)], { stdio: ["pipe", "pipe", "ignore"] }); + const peer = new CultNetPeer(Duplex.from({ readable: relay.stdout!, writable: relay.stdin! }), { wireContract: "cultnet.schema.v0" }); + return { peer, close: () => { relay.stdin!.end(); relay.kill(); } }; + }, +}; + +// ---- a client session: one connection, the frames it saw, one WriteTracker per write ----------------------------------------- + +class Session { + readonly frames: CultNetMessage[] = []; + readonly invalid: Error[] = []; + private readonly trackers = new Set(); + private serverReceipts = false; + private constructor(readonly target: Target, private readonly connection: Connection) { + connection.peer.on("message", (message: CultNetMessage) => { + if (message.schemaVersion === "cultnet.hello.v0") this.serverReceipts = helloAdvertisesWriteReceipts(message); + this.frames.push(message); + for (const tracker of this.trackers) tracker.frame(message, Date.now()); + }); + connection.peer.on("invalidMessage", (error: Error) => this.invalid.push(error)); + connection.peer.on("close", () => { for (const tracker of this.trackers) tracker.sessionEnd(Date.now()); }); + } + + static async open(target: Target, server: ServerProcess): Promise { + return new Session(target, await target.connect(server)); + } + + /** Declares the receipt capability and waits for the server's own hello. */ + async hello(versions: readonly string[] = CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS): Promise { + this.connection.peer.sendHello({ schemaVersion: "cultnet.hello.v0", runtimeId: "receipt-interop", runtimeKind: "cultmesh-ts", supportedMessageVersions: [...versions] }); + await waitFor(() => this.frames.some((frame) => frame.schemaVersion === "cultnet.hello.v0"), "the server's hello"); + return this.frames.find((frame) => frame.schemaVersion === "cultnet.hello.v0") as CultNetHelloMessage; + } + + put(messageId: string, recordKey: string, text: string, document: Record = {}): CultNetMessage { + return this.shard({ + schemaVersion: "cultnet.document_put_raw.v0", + messageId, + document: { + schemaId: SCHEMA_ID, recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", + payload: this.target.payload(text), ...document, + }, + }); + } + + delete(messageId: string, recordKey: string): CultNetMessage { + return this.shard({ schemaVersion: "cultnet.document_delete.v0", messageId, schemaId: SCHEMA_ID, recordKey }); + } + + private shard(message: unknown): CultNetMessage { + return { ...(message as object), ...this.target.shard } as CultNetMessage; + } + + /** + * Sends one write and settles it with its own tracker, windowed by what the server advertised (5000 ms against a receipt + * server, 500 ms otherwise). Only the tracker decides the outcome. + */ + async write(message: CultNetMessage): Promise { + const messageId = (message as { messageId: string }).messageId; + const tracker = new WriteTracker({ windowMs: resultWindowMs(this.serverReceipts) }); + this.trackers.add(tracker); + tracker.send(messageId, Date.now()); + this.connection.peer.send(message); + for (;;) { + tracker.tick(Date.now()); + const state = tracker.state(messageId)!; + if (state.outcome !== "pending") return state; + await sleep(Math.min(20, tracker.nextDeadlineMs(Date.now()) ?? 20)); + } + } + + /** Frames of `schemaVersion` that name `messageId`. */ + answers(messageId: string, schemaVersion: string): CultNetMessage[] { + return this.frames.filter((frame) => frame.schemaVersion === schemaVersion && (frame as { messageId?: string }).messageId === messageId); + } + + close(): void { this.connection.close(); } +} + +function refusal(settlement: WriteSettlement): CultNetErrorV1Message { + assert.equal(settlement.outcome, "refused", JSON.stringify(settlement)); + return (settlement as Extract).response as CultNetErrorV1Message; +} + +// ---- the scenarios, run against each server ----------------------------------------------------------------------------------- + +for (const target of [rust, csharp]) { + describe(`write receipt, TypeScript client against the real ${target.name} server`, () => { + let server: ServerProcess; + let session: Session | undefined; + before(async () => { server = await target.start(); }, { timeout: 180_000 }); + after(async () => { session?.close(); await server?.stop(); }); + + /** A fresh receipt session (hello sent, the server's hello read) for a scenario. */ + async function receiptSession(): Promise { + session?.close(); + session = await Session.open(target, server); + const hello = await session.hello(); + assert.ok(helloAdvertisesWriteReceipts(hello), "the server's hello lists both receipt frames"); + return session; + } + + test("hello listing both frames: the server answers with its own hello listing both, then a put is receipted after it", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + assert.equal(s.frames[0].schemaVersion, "cultnet.hello.v0"); + const settlement = await s.write(s.put("hello-1", "hello:a", "one")); + assert.deepEqual(settlement, { outcome: "admitted" }); + assert.deepEqual(s.frames.map((frame) => frame.schemaVersion), ["cultnet.hello.v0", "cultnet.write_admitted.v0"]); + }); + + test("a hello listing only one of the two frames does not opt in: no receipt, and the put is applied", { timeout: 30_000 }, async () => { + for (const listed of CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS) { + session?.close(); + session = await Session.open(target, server); + session.hello([listed]).catch(() => undefined); // a server may stay silent to a hello that does not opt in + await sleep(300); + const settlement = await session.write(session.put(`half-${listed}`, `half:${listed}`, "one")); + assert.deepEqual(settlement, { outcome: "unconfirmed", reason: "window_elapsed" }, `listing only ${listed}`); + assert.ok(!session.frames.some((frame) => frame.schemaVersion === "cultnet.write_admitted.v0" || frame.schemaVersion === "cultnet.error.v1")); + await waitFor(() => server.count("applied", `half:${listed}`) === 1, "the put to be applied"); + } + }); + + test("put admitted: the receipt echoes the write and the store has it once", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const settlement = await s.write(s.put("put-1", "put:a", "one")); + assert.deepEqual(settlement, { outcome: "admitted" }); + assert.deepEqual(s.answers("put-1", "cultnet.write_admitted.v0"), [ + { schemaVersion: "cultnet.write_admitted.v0", messageId: "put-1", operation: "put", schemaId: SCHEMA_ID, recordKey: "put:a" }, + ]); + await waitFor(() => server.count("applied", "put:a") === 1, "the put to be applied"); + }); + + // The Rust RUDP document server has no delete: its sink only accepts documents, and every delete on a receipt session is + // answered with a refusal (rudp_document_server.rs, deletes_on_a_receipt_session_are_refused_one_by_one). The spec + // (Server rules 1 to 3) has a delete admitted and receipted like a put. + test("delete admitted: the receipt says operation delete and the store drops the record", { + timeout: 30_000, + todo: target.name === "rust" ? "the Rust RUDP document server refuses every delete; there is no delete port in CultMeshRudpRawDocumentSink" : false, + }, async () => { + const s = await receiptSession(); + assert.deepEqual(await s.write(s.put("del-put", "del:a", "one")), { outcome: "admitted" }); + assert.deepEqual(await s.write(s.delete("del-1", "del:a")), { outcome: "admitted" }); + assert.deepEqual(s.answers("del-1", "cultnet.write_admitted.v0"), [ + { schemaVersion: "cultnet.write_admitted.v0", messageId: "del-1", operation: "delete", schemaId: SCHEMA_ID, recordKey: "del:a" }, + ]); + await waitFor(() => server.count("removed", "del:a") === 1, "the delete to be applied"); + }); + + test("refused write: an error.v1 carrying the write's id, and nothing is applied", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const response = refusal(await s.write(s.put("deny-1", "refuse:a", "one"))); + assert.equal(response.schemaVersion, "cultnet.error.v1"); + assert.equal(response.messageId, "deny-1"); + assert.equal(response.code, "write_not_authorized"); + // The session is kept: the next write on it is admitted. + assert.deepEqual(await s.write(s.put("deny-2", "deny:b", "one")), { outcome: "admitted" }); + assert.equal(server.count("applied", "refuse:a"), 0); + }); + + test("resend of the same id and content on the same session: re-receipted, applied once", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + assert.deepEqual(await s.write(s.put("resend-1", "resend:a", "one")), { outcome: "admitted" }); + assert.deepEqual(await s.write(s.put("resend-1", "resend:a", "one")), { outcome: "admitted" }); + const receipts = s.answers("resend-1", "cultnet.write_admitted.v0"); + assert.equal(receipts.length, 2); + assert.deepEqual(receipts[0], receipts[1]); + await sleep(200); // a second application would be printed by now + assert.equal(server.count("applied", "resend:a"), 1); + }); + + test("same id with different content: message_invalid naming the id, never a receipt, the first write untouched", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + assert.deepEqual(await s.write(s.put("clash-1", "clash:a", "one")), { outcome: "admitted" }); + const response = refusal(await s.write(s.put("clash-1", "clash:a", "other"))); + assert.equal(response.code, "message_invalid"); + assert.equal(response.messageId, "clash-1"); + assert.equal(response.details?.field, "messageId"); + assert.equal(s.answers("clash-1", "cultnet.write_admitted.v0").length, 1, "no receipt for the changed write"); + await sleep(200); + assert.equal(server.count("applied", "clash:a"), 1); + }); + + // Each case of the vectors' blankFields section: a document that names nothing is refused as an unreadable frame, with the + // write's id and the fixed text, and never applied; a document that is merely missing optional fields is admitted. + for (const blank of vectors.blankFields.cases as { name: string; set: Record; outcome: "admitted" | "refused" }[]) { + test(`blank-field vector ${blank.name}: ${blank.outcome}`, { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const recordKey = `blank:${blank.name}`; + const { payloadHex, ...fields } = blank.set; + const message = s.put(`blank-${blank.name}`, recordKey, "one", fields); + const sent = (message as unknown as { document: Record }).document; + for (const [key, value] of Object.entries(fields)) if (value === null) delete sent[key]; // nil means absent on the wire here + if (payloadHex === "") sent.payload = new Uint8Array(); + const settlement = await s.write(message); + if (blank.outcome === "admitted") { + assert.deepEqual(settlement, { outcome: "admitted" }); + await waitFor(() => server.count("applied", recordKey) === 1, "the put to be applied"); + return; + } + const response = refusal(settlement); + assert.equal(response.code, "message_invalid"); + assert.equal(response.error, UNREADABLE); + assert.equal(response.details?.field, "message"); + assert.equal(response.messageId, `blank-${blank.name}`); + await sleep(200); + assert.equal(server.count("applied", recordKey), 0); + assert.equal(server.count("applied", "blank:"), 0); + }); + } + + test("a delete that names nothing is refused as unreadable with its id", { timeout: 30_000 }, async () => { + const s = await receiptSession(); + const message = s.delete("blank-del", "blank:del"); + (message as unknown as { schemaId: string }).schemaId = ""; + const response = refusal(await s.write(message)); + assert.equal(response.code, "message_invalid"); + assert.equal(response.error, UNREADABLE); + assert.equal(response.details?.field, "message"); + assert.equal(response.messageId, "blank-del"); + }); + + test("legacy client (no hello): no receipt frames, the write is applied, and its refusal is the old error.v0", { timeout: 30_000 }, async () => { + session?.close(); + session = await Session.open(target, server); + assert.deepEqual(await session.write(session.put("legacy-1", "legacy:a", "one")), { outcome: "unconfirmed", reason: "window_elapsed" }); + await waitFor(() => server.count("applied", "legacy:a") === 1, "the put to be applied"); + const refused = (await session.write(session.put("legacy-2", "refuse:legacy", "one"))) as Extract; + assert.equal(refused.outcome, "refused"); + assert.equal(refused.response.schemaVersion, "cultnet.error.v0"); + assert.deepEqual(session.frames.map((frame) => frame.schemaVersion), ["cultnet.error.v0"]); + assert.equal(session.invalid.length, 0); + }); + }); +} + +// ---- the one-shot writer, which is the real cultmesh-ts client path, against the Rust RUDP server ---------------------------- +// (The C# server speaks LiteNetLib, not RUDP: no RUDP client path reaches it.) + +describe("write receipt, cultmesh-ts publishRudpDocumentOnce against the real rust server", () => { + const note = defineDocumentType({ type: "receipt.interop_note", schemaId: SCHEMA_ID, schema: z.object({ text: z.string() }), name: "text" }); + const binding = defineCultNetDocumentBinding({ definition: note }); + let server: ServerProcess; + before(async () => { server = await rust.start(); }, { timeout: 180_000 }); + after(async () => { await server?.stop(); }); + const endpoint = () => ({ host: "127.0.0.1", port: server.port, uri: `rudp://127.0.0.1:${server.port}` }); + + test("writeReceipts: the put resolves admitted on the receipt, after the server applied it", { timeout: 30_000 }, async () => { + const outcome = await CultMesh.publishRudpDocumentOnce("receipt-interop", nextConnectionId++, endpoint(), binding, "once:a", { text: "one" }, { writeReceipts: true, messageId: "once-1" }); + assert.equal(outcome, "admitted"); + await waitFor(() => server.count("applied", "once:a") === 1, "the put to be applied"); + }); + + test("writeReceipts: a refused put rejects with the server's error.v1 naming the put", { timeout: 30_000 }, async () => { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("receipt-interop", nextConnectionId++, endpoint(), binding, "refuse:once", { text: "one" }, { writeReceipts: true, messageId: "once-2" }), + (error: unknown) => { + assert.ok(error instanceof CultMeshRudpWriteRefusedError); + assert.equal(error.response.schemaVersion, "cultnet.error.v1"); + assert.equal((error.response as CultNetErrorV1Message).messageId, "once-2"); + return true; + }, + ); + }); + + test("without writeReceipts the server stays silent: the put is unconfirmed, not admitted, though the server applied it", { timeout: 30_000 }, async () => { + await assert.rejects( + CultMesh.publishRudpDocumentOnce("receipt-interop", nextConnectionId++, endpoint(), binding, "once:legacy", { text: "one" }, { messageId: "once-3" }), + (error: unknown) => error instanceof CultMeshRudpWriteUnconfirmedError && error.reason === "window_elapsed", + ); + await waitFor(() => server.count("applied", "once:legacy") === 1, "the put to be applied"); + }); +}); From 04207007d6b51d3be27b547300e1e2aac1718244 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 07:45:35 +0000 Subject: [PATCH 05/40] interop test: read the store under the record key the blank-field vector set A vector that sets recordKey (zero_width_space_record_key_is_a_name) replaces the harness key, so the admitted case waited for blank: and timed out against the Rust server, and the refused record-key cases counted applications under a key the server never saw. Use the key the document carried. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-ts/test/interop/write-receipt-interop.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts index 8a646a50..12bd4394 100644 --- a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts +++ b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts @@ -331,8 +331,9 @@ for (const target of [rust, csharp]) { for (const blank of vectors.blankFields.cases as { name: string; set: Record; outcome: "admitted" | "refused" }[]) { test(`blank-field vector ${blank.name}: ${blank.outcome}`, { timeout: 30_000 }, async () => { const s = await receiptSession(); - const recordKey = `blank:${blank.name}`; const { payloadHex, ...fields } = blank.set; + // A case that sets recordKey replaces the harness key, so the store is read under the key the server actually saw. + const recordKey = typeof fields.recordKey === "string" ? fields.recordKey : `blank:${blank.name}`; const message = s.put(`blank-${blank.name}`, recordKey, "one", fields); const sent = (message as unknown as { document: Record }).document; for (const [key, value] of Object.entries(fields)) if (value === null) delete sent[key]; // nil means absent on the wire here @@ -350,7 +351,6 @@ for (const target of [rust, csharp]) { assert.equal(response.messageId, `blank-${blank.name}`); await sleep(200); assert.equal(server.count("applied", recordKey), 0); - assert.equal(server.count("applied", "blank:"), 0); }); } From 4f6ee0db835aa1e6b7a5ce6d7e175ea463e75792 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:22:55 +0000 Subject: [PATCH 06/40] Rust RUDP server: pin the refusal of blank schemaId/recordKey on receipt sessions heimdall-low-rust-keys-untested. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index be667e72..e37ce9d6 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -2983,6 +2983,75 @@ fn only_reliable_non_sequenced_writes_are_accepted_on_a_receipt_session() -> Res Ok(()) } +/// `bytes` with the MessagePack fixstr `from` replaced by the fixstr `to`, which may be blank. +fn swap_fixstr(bytes: &[u8], from: &str, to: &str) -> Vec { + assert!(from.len() < 32 && to.len() < 32); + let mut needle = vec![0xa0 | from.len() as u8]; + needle.extend_from_slice(from.as_bytes()); + let at = bytes.windows(needle.len()).position(|window| window == needle).expect("marker is in the frame"); + let mut out = bytes[..at].to_vec(); + out.push(0xa0 | to.len() as u8); + out.extend_from_slice(to.as_bytes()); + out.extend_from_slice(&bytes[at + needle.len()..]); + out +} + +/// A `schemaId` or `recordKey` that is present but empty or only whitespace names nothing: the write is refused +/// `message_invalid` like a frame that does not read (field "message", the write's id), never reaches the store, +/// and the session goes on (write-receipt.md, rule 5; vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`). +#[test] +fn a_write_whose_schema_id_or_record_key_names_nothing_is_refused_and_never_handed_to_the_store() -> Result<()> { + let blanks = ["", " ", "\t \n"]; + let mut connection_id = 300; + for blank in blanks { + for blank_field in ["schemaId", "recordKey"] { + for operation in ["put", "delete"] { + connection_id += 1; + let (schema_id, record_key) = match blank_field { + "schemaId" => (blank, "note:1"), + _ => ("test.raw.v1", blank), + }; + // The encoder refuses a blank field, so encode a marker and swap the blank into the bytes. + let bad = if operation == "put" { + let mut record = document("zz-key-zz", vec![1, 2, 3]); + record.schema_id = "zz-schema-zz".into(); + encode_cultnet_message_to_vec( + &CultNetMessage::DocumentPutRaw { message_id: "bad-1".into(), document: record }, + CultNetWireContract::CultNetSchemaV0, + )? + } else { + encode_cultnet_message_to_vec( + &CultNetMessage::DocumentDelete { + message_id: "bad-1".into(), + schema_id: "zz-schema-zz".into(), + record_key: "zz-key-zz".into(), + }, + CultNetWireContract::CultNetSchemaV0, + )? + }; + let bad = swap_fixstr(&bad, "zz-schema-zz", schema_id); + let bad = swap_fixstr(&bad, "zz-key-zz", record_key); + let label = format!("{operation} with {blank_field} {blank:?}"); + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, connection_id)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + peer.send_reliable("schema", bad)?; + send_reliable(&mut peer, &put("good-1", "note:2", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("bad-1".into())), "{label}"); + assert_eq!(error_text_and_field(&answers[0]).1, Some("message".into()), "{label}"); + assert_eq!(answers[1], admitted("good-1", "note:2"), "{label}: the session goes on"); + let state = sink.0.lock().unwrap(); + assert_eq!(state.authorized, vec!["good-1".to_string()], "{label}: the store was never asked about the blank write"); + assert_eq!(state.receipts.len(), 1, "{label}: only the good write was applied"); + } + } + } + Ok(()) +} + /// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured /// from the refusal and not from the packet before it. #[test] From bd2aff8ed0cdd76d6acc673dff5a0fea8fc10d72 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 21:23:07 +0000 Subject: [PATCH 07/40] Rust RUDP server: decide the sequenced-channel refusal per frame A sequenced packet's arrival can release held ordered frames in the same delivery; refuse only the packet's own frame. heimdall-low-refuse-sequenced-per-frame. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 4 +- .../cultmesh-rs/tests/rudp_document_server.rs | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index f97b2578..12fdf3e6 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -825,7 +825,9 @@ where if frame.channel_id != "schema" { continue; } - if refuse_sequenced { + // `delivered` also holds the ordered frames this packet's arrival released from the hold; those were + // not sent on the sequenced channel. They sit above the packet's own sequence, its own frame at or below. + if refuse_sequenced && frame.sequence <= packet.sequence { if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = self.refuse_unreliable_write(key, &frame.payload)? { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index e37ce9d6..b6663396 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3052,6 +3052,51 @@ fn a_write_whose_schema_id_or_record_key_names_nothing_is_refused_and_never_hand Ok(()) } +/// The sequenced-channel refusal belongs to the frame that arrived sequenced. A reliable-sequenced packet that +/// fills the gap an earlier ordered packet was held behind delivers both frames in one go: the sequenced write is +/// refused, the ordered write it released is admitted. +#[test] +fn a_sequenced_packet_releasing_a_held_ordered_write_refuses_only_its_own() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 250)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let mut packet_for = |peer: &mut RawPeer, id: &str, ordered: bool, sequenced: bool| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered, sequenced, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let sequenced = packet_for(&mut peer, "s1", false, true)?; + let ordered = packet_for(&mut peer, "o1", true, false)?; + // The ordered write arrives first, one sequence ahead of the gap, and is held. + for packet in &ordered { + peer.send_packet(packet)?; + } + server.poll_once()?; + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "the ordered write waits behind the gap"); + for packet in &sequenced { + peer.send_packet(packet)?; + } + server.poll_once()?; + let mut answers = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + let refusals: Vec<_> = answers.iter().filter(|a| matches!(a, CultNetMessage::ErrorV1 { .. })).collect(); + assert_eq!(refusals.len(), 1, "only the sequenced write is refused: {answers:?}"); + assert_eq!(refusal_of(refusals[0]), (Some(CultNetErrorCode::MessageInvalid), Some("s1".into()))); + assert!(answers.contains(&admitted("o1", "note:o1")), "the released ordered write is admitted: {answers:?}"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "and applied"); + Ok(()) +} + /// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured /// from the refusal and not from the packet before it. #[test] From 4e89f07bff194851d518c7cb921b27a8f3fe375a Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:13:24 +0000 Subject: [PATCH 08/40] Rust RUDP server: answer a superseded sequenced write; order frame and packet by serial arithmetic Superseded packet: a reliable-sequenced packet that arrives after a newer one delivers no frame, yet is acknowledged and never resent, so its write went unanswered. The test (red first: only "new" was refused, "old" got nothing) showed the drop happens in the session below frame decoding, but an unfragmented packet carries its whole frame, so the server can read the id from the packet payload. Chosen: refuse it from the packet payload. A superseded fragment of a larger write has no readable id in the packet, so write-receipt.md now says that write may go unanswered and the client's unconfirmed outcome covers it. Ordering: frame.sequence <= packet.sequence is replaced by sequence_at_or_before, serial arithmetic modulo 2^32. Deviation from the brief: cultnet-rs' at_or_before carries a 4096-sequence receive window made for duplicate detection; a frame's first fragment may lie further back than that, so reusing it would leave such a sequenced write unrefused and applied. The new test pins that, and a hand mutant with the window fails it. A wrapped sequence cannot reach this compare through the session today (it is judged a duplicate), so the unit test pins the helper, not an end-to-end wrap. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- .../cultmesh-rs/src/rudp_document_server.rs | 33 +++++- .../cultmesh-rs/tests/rudp_document_server.rs | 102 ++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index d3dc2614..b74411ea 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -38,7 +38,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. + - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 12fdf3e6..a0ab51ac 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -821,13 +821,26 @@ where } let mut kept_rejection = None; + // A superseded packet delivers no frame, yet it was acknowledged and will not be resent, so its write would + // go unanswered. A packet that carries its whole frame is answered from its own payload. A superseded + // fragment cannot be: the write's id may lie in a fragment this packet does not hold. + if refuse_sequenced + && packet.fragment_count == 0 + && !result.delivered.iter().any(|frame| frame.channel_id == "schema" && frame.sequence == packet.sequence) + { + if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = + self.refuse_unreliable_write(key, &packet.payload)? + { + kept_rejection = Some(rejection); + } + } for frame in result.delivered { if frame.channel_id != "schema" { continue; } // `delivered` also holds the ordered frames this packet's arrival released from the hold; those were // not sent on the sequenced channel. They sit above the packet's own sequence, its own frame at or below. - if refuse_sequenced && frame.sequence <= packet.sequence { + if refuse_sequenced && sequence_at_or_before(frame.sequence, packet.sequence) { if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = self.refuse_unreliable_write(key, &frame.payload)? { @@ -1816,6 +1829,13 @@ where #[cfg(test)] mod send_failure_tests; +/// Whether `sequence` is at or before `mark` in serial order (RFC 1982): the 32-bit space is compared modulo 2^32, +/// so a frame that began before the wrap and a packet after it still order correctly. The receive window of +/// `cultnet_rs`' duplicate test does not belong here: a frame's first fragment may lie any distance back. +fn sequence_at_or_before(sequence: u32, mark: u32) -> bool { + mark.wrapping_sub(sequence) < 1 << 31 +} + fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.max_sessions == 0 { return Err(anyhow!("max_sessions must be greater than zero")); @@ -1931,4 +1951,15 @@ mod tests { .unwrap(); assert_eq!(hex(&bytes), MESSAGE_INVALID_HEX); } + + #[test] + fn sequence_order_holds_across_the_wrap_and_beyond_the_receive_window() { + assert!(sequence_at_or_before(7, 7)); + assert!(sequence_at_or_before(100, 5_200), "further back than the receive window"); + assert!(!sequence_at_or_before(5_200, 100)); + assert!(sequence_at_or_before(u32::MAX - 1, 1), "begun before the wrap, ended after it"); + assert!(!sequence_at_or_before(1, u32::MAX - 1), "after the wrap is later than before it"); + assert!(sequence_at_or_before(u32::MAX, 0), "the wrap itself"); + assert!(!sequence_at_or_before(0, u32::MAX)); + } } diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index b6663396..d9b42abb 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3097,6 +3097,108 @@ fn a_sequenced_packet_releasing_a_held_ordered_write_refuses_only_its_own() -> R Ok(()) } +/// The refusal of a write on a sequenced channel does not depend on the packet being the newest. A reliable-sequenced +/// packet that arrives after a newer one on its channel is superseded: the session delivers no frame for it, yet +/// the packet was acknowledged and will not be resent, so the server answers it from the packet's own payload. +#[test] +fn a_superseded_sequenced_write_is_still_refused() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 253)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let packet_for = |peer: &mut RawPeer, id: &str| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: false, sequenced: true, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let older = packet_for(&mut peer, "old")?; + let newer = packet_for(&mut peer, "new")?; + let mut refused_ids = Vec::new(); + for packets in [&newer, &older] { + for packet in packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + let answer = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + if matches!(answer, CultNetMessage::ErrorV1 { .. }) { + refused_ids.push(refusal_of(&answer).1.expect("a refusal names its write")); + } + } + } + } + assert_eq!(refused_ids, vec!["new".to_string(), "old".to_string()], "the superseded write is answered too"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "and nothing was applied"); + Ok(()) +} + +/// The refusal reaches a sequenced write however far apart its fragments' sequences sit: other traffic can take the +/// sequences between the first fragment and the last, further back than the duplicate test's receive window. +#[test] +fn a_sequenced_write_whose_fragments_span_more_than_the_receive_window_is_refused() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 254)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let payload = encode_cultnet_message_to_vec(&put("far", "note:far", vec![1; 64]), CultNetWireContract::CultNetSchemaV0)?; + let mut packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: false, sequenced: true, now_ms: 3, reliable_expire_after_ms: None }, + Some(32), + )?; + let last = packets.pop().expect("the write is fragmented"); + assert!(!packets.is_empty()); + let mut next = last.sequence; + let mut refused_ids = Vec::new(); + for packet in &packets { + peer.send_packet(packet)?; + server.poll_once()?; + } + // 5,000 reliable packets on another channel, in order, take the sequences between the fragments. + for count in 0..5_000 { + let mut filler = packets[0].clone(); + filler.channel_id = "filler".into(); + filler.sequenced = false; + filler.fragment_id = 0; + filler.fragment_index = 0; + filler.fragment_count = 0; + filler.payload = vec![0]; + filler.sequence = next; + next += 1; + peer.send_packet(&filler)?; + server.poll_once()?; + if count % 100 == 99 { + // Read what the server sent, so the socket buffer never holds back the refusal. + for packet in peer.drain() { + peer.session.receive(&packet, 4)?; + } + } + } + let mut last = last; + last.sequence = next; + peer.send_packet(&last)?; + server.poll_once()?; + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + let answer = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + if matches!(answer, CultNetMessage::ErrorV1 { .. }) { + refused_ids.push(refusal_of(&answer).1.expect("a refusal names its write")); + } + } + } + assert_eq!(refused_ids, vec!["far".to_string()]); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "and nothing was applied"); + Ok(()) +} + /// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured /// from the refusal and not from the packet before it. #[test] From 34cceedf9841b57f9bc01c7bad5e43199ec7e2ff Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 22:15:34 +0000 Subject: [PATCH 09/40] Rust RUDP server: pin the superseded-write refusal against a releasing packet and the half-space bound cargo-mutants on the previous commit missed two mutants: `&&` to `||` in the superseded-packet check (a packet that also releases another write hid it) and `<` to `<=` at the half-space bound of sequence_at_or_before. A test and two assertions kill them; the rerun catches 14 of 15 (1 unviable). Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 2 + .../cultmesh-rs/tests/rudp_document_server.rs | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index a0ab51ac..1e729407 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -1961,5 +1961,7 @@ mod tests { assert!(!sequence_at_or_before(1, u32::MAX - 1), "after the wrap is later than before it"); assert!(sequence_at_or_before(u32::MAX, 0), "the wrap itself"); assert!(!sequence_at_or_before(0, u32::MAX)); + assert!(sequence_at_or_before(0, (1 << 31) - 1), "just inside half the space"); + assert!(!sequence_at_or_before(0, 1 << 31), "half the space away is not before"); } } diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index d9b42abb..6ca0007a 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3138,6 +3138,50 @@ fn a_superseded_sequenced_write_is_still_refused() -> Result<()> { Ok(()) } +/// A superseded sequenced packet is answered even when its arrival fills a gap and releases another write: the +/// frames that arrive with it do not stand in for its own. +#[test] +fn a_superseded_sequenced_write_is_refused_while_it_releases_a_held_ordered_write() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 255)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let packet_for = |peer: &mut RawPeer, id: &str, ordered: bool, sequenced: bool| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered, sequenced, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let older = packet_for(&mut peer, "old", false, true)?; + let ordered = packet_for(&mut peer, "o1", true, false)?; + let newer = packet_for(&mut peer, "new", false, true)?; + let mut answers = Vec::new(); + for packets in [&newer, &ordered, &older] { + for packet in packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + } + let mut refused: Vec = answers + .iter() + .filter(|a| matches!(a, CultNetMessage::ErrorV1 { .. })) + .map(|a| refusal_of(a).1.expect("a refusal names its write")) + .collect(); + refused.sort(); + assert_eq!(refused, vec!["new".to_string(), "old".to_string()], "{answers:?}"); + assert!(answers.contains(&admitted("o1", "note:o1")), "the released ordered write is admitted: {answers:?}"); + Ok(()) +} + /// The refusal reaches a sequenced write however far apart its fragments' sequences sit: other traffic can take the /// sequences between the first fragment and the last, further back than the duplicate test's receive window. #[test] From 6c441d1cda7780a90ecbff2dacaa3f0e7ac7e542 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 09:12:59 +0000 Subject: [PATCH 10/40] Rust RUDP server: answer a sequenced write once, not once per send A repeat of a packet the session already took, and a packet too far ahead to be remembered, delivered no frame and so drew the superseded-write refusal again on every send, breaking write-receipt rule 1. The refusal now applies only to the first packet the session takes for a sequence; CultNetRudpSession::has_received exposes that test. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 9 +++ .../cultmesh-rs/tests/rudp_document_server.rs | 58 +++++++++++++++++++ packages/cultnet-rs/src/rudp.rs | 6 ++ 3 files changed, 73 insertions(+) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 1e729407..9aae5be9 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -780,6 +780,13 @@ where return Ok(CultMeshRudpPollOutcome::Handled); } + // Whether this packet is the first the session takes for its sequence: a repeat of an answered write, or a + // packet too far ahead to be remembered, is not, and the write is not answered for it again. + let first_receipt = packet.reliable + && self + .sessions + .get(&key) + .is_some_and(|entry| !entry.session.has_received(packet.sequence)); let result = { let entry = self .sessions @@ -825,6 +832,8 @@ where // go unanswered. A packet that carries its whole frame is answered from its own payload. A superseded // fragment cannot be: the write's id may lie in a fragment this packet does not hold. if refuse_sequenced + && first_receipt + && self.sessions.get(&key).is_some_and(|entry| entry.session.has_received(packet.sequence)) && packet.fragment_count == 0 && !result.delivered.iter().any(|frame| frame.channel_id == "schema" && frame.sequence == packet.sequence) { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 6ca0007a..60596ee5 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -3138,6 +3138,64 @@ fn a_superseded_sequenced_write_is_still_refused() -> Result<()> { Ok(()) } +/// One answer per write (write-receipt.md, rule 1): a sequenced packet the server already answered is answered +/// once, however often the peer sends it again, and a packet too far ahead to be received draws nothing until it +/// is received (the peer resends it, and the answer comes then). +#[test] +fn a_repeated_or_far_ahead_sequenced_write_draws_one_answer_not_one_per_send() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 256)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let packet_for = |peer: &mut RawPeer, id: &str| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, &format!("note:{id}"), vec![1]), CultNetWireContract::CultNetSchemaV0)?; + peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: false, sequenced: true, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + ) + }; + let refusals_of = |peer: &mut RawPeer| -> Result> { + let mut ids = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + let answer = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + if matches!(answer, CultNetMessage::ErrorV1 { .. }) { + ids.push(refusal_of(&answer).1.expect("a refusal names its write")); + } + } + } + Ok(ids) + }; + // The same packet, sent three times (a resend whose acknowledgement was lost), is refused once. + let once = packet_for(&mut peer, "dup")?; + let mut refused = Vec::new(); + for _ in 0..3 { + for packet in &once { + peer.send_packet(packet)?; + } + server.poll_once()?; + refused.extend(refusals_of(&mut peer)?); + } + assert_eq!(refused, vec!["dup".to_string()], "a repeat of an answered write is not answered again"); + + // A packet far past the receive window is not received: no answer yet, and none per resend. + let ahead = packet_for(&mut peer, "ahead")?; + let mut far = ahead[0].clone(); + far.sequence = once[0].sequence + 5_000; + let mut refused = Vec::new(); + for _ in 0..3 { + peer.send_packet(&far)?; + server.poll_once()?; + refused.extend(refusals_of(&mut peer)?); + } + assert_eq!(refused, Vec::::new(), "an unreceived packet draws no answer"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + Ok(()) +} + /// A superseded sequenced packet is answered even when its arrival fills a gap and releases another write: the /// frames that arrive with it do not stand in for its own. #[test] diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index eae7a2ee..427a38c1 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -1277,6 +1277,12 @@ impl CultNetRudpSession { } } + /// Whether `receive` has taken this reliable sequence: delivered, held, superseded or a duplicate. A packet + /// `receive` refused unremembered (too far ahead, or before the handshake) is not, and its sender resends it. + pub fn has_received(&self, sequence: u32) -> bool { + self.was_received(sequence) + } + /// The one duplicate test: true for a sequence in the window that was /// received, for one below the window, and for one at or before the /// watermark. The watermark starts at the handshake's seed, so a frame the From 636b387cf10550fcb335344f679f3f1879d6e8c8 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 09:13:31 +0000 Subject: [PATCH 11/40] write-receipt spec: a repeated or unreceived sequenced packet draws no second answer Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index b74411ea..234fdf93 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -38,7 +38,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session. - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case. - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`. - - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. + - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. A repeat of a packet the server already took, or a packet too far ahead to be taken, draws no further answer (rule 1). - Past the session (or a server's longer retention, if it keeps one) a resend is a new write. 6. **Forwarded writes.** A server that forwards a non-primary write answers the original peer only with what the primary answered: the primary's receipt becomes `write_admitted` with the original id, its refusal becomes `error.v1` with the original id. If the forward gets no answer the server relays nothing: no receipt, no error, no synthesized refusal. The client's own rule below then reports the write as unconfirmed. A forwarder never turns an absent or ambiguous primary answer into an admission. 7. **Looks alike.** An admitted and a refused write differ in `schemaVersion`; a receipt has no `error`, an `error.v1` has no admission. A transport ack is neither. From 8c31933e108b00240708a32a9b9db12a4644e0c3 Mon Sep 17 00:00:00 2001 From: Libby Date: Mon, 9 Nov 2026 10:38:29 +0000 Subject: [PATCH 12/40] cultnet-py: the write-receipt vector sections include the file's counts Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-py/tests/test_write_receipt.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 32c5f03a..98e74287 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -19,7 +19,7 @@ from cultnet_py import ( ) # Decoder-side conformance with contracts/cultnet/interop/write-receipt-vectors.json: `contentHash`, `wire` and -# `schemaInvalid`. `dedupe` and `blankFields` are server obligations and `scenarios` is the client write tracker; +# `schemaInvalid`. `dedupe` and `blankFields` are server obligations and `scenarios` is the client write tracker; `counts` is the file's own tally; # this runtime has neither a receipt server nor a write client, so it replays none of them. _INTEROP = Path(__file__).resolve().parents[3] / "contracts/cultnet/interop" _VECTORS = json.loads((_INTEROP / "write-receipt-vectors.json").read_text(encoding="utf-8")) @@ -57,11 +57,17 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) self.assertEqual(len(_VECTORS["schemaInvalid"]), 7) + # The file declares how many of each it holds; a loader that drops entries disagrees with it. + counts = _VECTORS["counts"] + self.assertEqual(counts["wire"], len(_VECTORS["wire"])) + self.assertEqual(counts["schemaInvalid"], len(_VECTORS["schemaInvalid"])) + self.assertEqual(counts["contentHashCases"], len(_VECTORS["contentHash"]["cases"])) + self.assertEqual(counts["scenarioCases"], len(_VECTORS["scenarios"]["cases"])) def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: From 1704fb5060f392395c9df55522bcf45f943c2f6e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:05:57 +0000 Subject: [PATCH 13/40] cultmesh-ts: test:interop builds cultcache-ts and cultnet-ts (with contracts) first Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-ts/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index 0fef7419..39e0abba 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -25,7 +25,7 @@ "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --expose-gc --test dist-test/test/*.test.js", - "test:interop": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/write-receipt-interop.test.js" + "test:interop": "npm --prefix ../.. run build --workspace @gamecult/cultcache-ts --workspace @gamecult/cultnet-ts && node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test --test-force-exit dist-test/test/interop/write-receipt-interop.test.js" }, "keywords": [ "cultmesh", From b8e40dc372660774535f021c39a70aa7b968679e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:10:42 +0000 Subject: [PATCH 14/40] interop test: pace C# relay connections under the server's per-IP per-minute limit Co-Authored-By: Claude Sonnet 5.5 --- .../test/interop/write-receipt-interop.test.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts index 12bd4394..092d5ffb 100644 --- a/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts +++ b/packages/cultmesh-ts/test/interop/write-receipt-interop.test.ts @@ -16,7 +16,7 @@ import { execFile, spawn, spawnSync, type ChildProcess } from "node:child_proces import { readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { Duplex } from "node:stream"; -import { after, before, describe, test } from "node:test"; +import { after, before, beforeEach, describe, test } from "node:test"; import { promisify } from "node:util"; import { encode } from "@msgpack/msgpack"; import { z } from "zod"; @@ -134,6 +134,20 @@ const rust: Target = { }, }; +// The C# server admits 30 connection requests per IP per minute (Server.MaxConnectionAttemptsPerMinute) and answers the rest with +// no hello. Every scenario wants a fresh session, so before each C# test the harness waits until the last minute holds at most +// CONNECTIONS_PER_MINUTE connections, leaving room for the two a test may open. The server's limit stays as it is. +const CONNECTIONS_PER_MINUTE = 24; +const connectedAt: number[] = []; +async function awaitConnectionBudget(): Promise { + for (;;) { + const now = Date.now(); + while (connectedAt.length > 0 && now - connectedAt[0] >= 61_000) connectedAt.shift(); + if (connectedAt.length <= CONNECTIONS_PER_MINUTE) return; + await sleep(connectedAt[0] + 61_000 - now); + } +} + const csharp: Target = { name: "csharp", shard: { shardId: "receipt-interop-shard", shardEpoch: 1 }, @@ -143,6 +157,7 @@ const csharp: Target = { return launch(dotnet, [csharpDll, "receipt-serve"]); }, async connect(server) { + connectedAt.push(Date.now()); const relay = spawn(dotnet, [csharpDll, "receipt-relay", "--port", String(server.port)], { stdio: ["pipe", "pipe", "ignore"] }); const peer = new CultNetPeer(Duplex.from({ readable: relay.stdout!, writable: relay.stdin! }), { wireContract: "cultnet.schema.v0" }); return { peer, close: () => { relay.stdin!.end(); relay.kill(); } }; @@ -235,6 +250,7 @@ for (const target of [rust, csharp]) { let session: Session | undefined; before(async () => { server = await target.start(); }, { timeout: 180_000 }); after(async () => { session?.close(); await server?.stop(); }); + beforeEach(async () => { if (target === csharp) await awaitConnectionBudget(); }, { timeout: 90_000 }); /** A fresh receipt session (hello sent, the server's hello read) for a scenario. */ async function receiptSession(): Promise { From 96c5fb1ca4dfa1e4523201573af49cb108082a00 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:12:38 +0000 Subject: [PATCH 15/40] interop peer: C# store holds the spec's admitted schema-id names (compatible ids) Co-Authored-By: Claude Sonnet 5.5 --- tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs index 75eaf4da..6871ea00 100644 --- a/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs +++ b/tests/GameCult.Networking.InteropPeer/ReceiptHarness.cs @@ -14,7 +14,9 @@ using R3; // ordered payload, and on the pipe it is a 4-byte big-endian length followed by the frame. namespace GameCult.Networking.InteropPeer { - [CultDocument("receipt.interop_note", ReceiptHarness.SchemaId)] + // CompatibleSchemaIds are the names the spec's blankFields vectors admit (U+200B, U+001C, and a name padded with NBSP and EM SPACE): + // a name is any string that is not blank, so the store must hold a record put under one of them. + [CultDocument("receipt.interop_note", ReceiptHarness.SchemaId, CompatibleSchemaIds = ["\u200b", "\u001c", "\u00a0a\u2003"])] [MessagePack.MessagePackObject] public sealed class ReceiptInteropNote { From 4571a0d4e7f1e9b0da9ddfec962c0dc7a7fa1366 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:39:51 +0000 Subject: [PATCH 16/40] tests first: write_admitted messageId rule vectors (failing) Pins Heimdall's rule: a write_admitted messageId needs a Unicode letter or number and no Default_Ignorable_Code_Point, compared raw. Shared vectors plus a runner test in TS, Python, Kotlin, C# and Rust. No decoder changed; the refuse vectors fail. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 277 +++++++++++++++++- contracts/cultnet/interop/write-receipt.md | 4 +- .../gamecult/cultmesh/WriteReceiptVectors.kt | 23 +- .../cultnet-py/tests/test_write_receipt.py | 27 +- packages/cultnet-rs/tests/write_receipt.rs | 48 +++ .../cultnet-ts/test/write-receipt.test.ts | 21 ++ .../CultNetWriteAdmittedMessageIdTests.cs | 58 ++++ 7 files changed, 454 insertions(+), 4 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 397fbad6..a37d9660 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -149,6 +149,280 @@ { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" } ] }, + "admittedMessageId": { + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Scope: write_admitted only; error.v1 is untouched.", + "cases": [ + { + "name": "refuse_braille_blank_2800", + "codePoints": [ + "2800" + ], + "expect": "refuse", + "why": "U+2800 is Default_Ignorable and not a letter or number" + }, + { + "name": "refuse_hangul_filler_3164", + "codePoints": [ + "3164" + ], + "expect": "refuse", + "why": "U+3164 Default_Ignorable (a letter by category Lo, so DI decides)" + }, + { + "name": "refuse_hangul_choseong_filler_115F", + "codePoints": [ + "115F" + ], + "expect": "refuse", + "why": "U+115F Default_Ignorable (category Lo)" + }, + { + "name": "refuse_halfwidth_hangul_filler_FFA0", + "codePoints": [ + "FFA0" + ], + "expect": "refuse", + "why": "U+FFA0 Default_Ignorable (category Lo)" + }, + { + "name": "refuse_combining_grapheme_joiner_034F", + "codePoints": [ + "034F" + ], + "expect": "refuse", + "why": "U+034F Default_Ignorable" + }, + { + "name": "refuse_combining_mark_alone_0300", + "codePoints": [ + "0300" + ], + "expect": "refuse", + "why": "a combining mark is not a letter or number" + }, + { + "name": "refuse_private_use_E000", + "codePoints": [ + "E000" + ], + "expect": "refuse", + "why": "private use is neither a letter nor a number" + }, + { + "name": "refuse_unassigned_0378", + "codePoints": [ + "0378" + ], + "expect": "refuse", + "why": "unassigned since Unicode 1.1, neither a letter nor a number" + }, + { + "name": "refuse_lone_surrogate_D800", + "codePoints": [ + "D800" + ], + "expect": "refuse", + "why": "not a Unicode scalar value; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str, so the decoder must refuse the frame. Rust, C#, Kotlin and Python build the frame from wireHex; a runtime whose strings can hold a lone surrogate (TypeScript) may also build it from the code point", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "refuse_control_nul_0000", + "codePoints": [ + "0000" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_bel_0007", + "codePoints": [ + "0007" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_unit_separator_001F", + "codePoints": [ + "001F" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_delete_007F", + "codePoints": [ + "007F" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_next_line_0085", + "codePoints": [ + "0085" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_empty", + "codePoints": [], + "expect": "refuse", + "why": "empty" + }, + { + "name": "refuse_whitespace_only_space_0020", + "codePoints": [ + "0020" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_whitespace_only_nbsp_00A0", + "codePoints": [ + "00A0" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_whitespace_only_ideographic_space_3000", + "codePoints": [ + "3000" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_letter_zwsp_letter", + "codePoints": [ + "0061", + "200B", + "0062" + ], + "expect": "refuse", + "why": "a Default_Ignorable (U+200B) beside letters" + }, + { + "name": "refuse_letter_cgj", + "codePoints": [ + "0061", + "034F" + ], + "expect": "refuse", + "why": "a Default_Ignorable (U+034F) after a letter" + }, + { + "name": "refuse_letter_soft_hyphen", + "codePoints": [ + "0061", + "00AD" + ], + "expect": "refuse", + "why": "U+00AD is Default_Ignorable" + }, + { + "name": "refuse_letter_zwj", + "codePoints": [ + "0061", + "200D" + ], + "expect": "refuse", + "why": "U+200D is Default_Ignorable" + }, + { + "name": "refuse_digit_mongolian_vowel_separator", + "codePoints": [ + "0031", + "180E" + ], + "expect": "refuse", + "why": "U+180E is Default_Ignorable since Unicode 6.3" + }, + { + "name": "refuse_digit_word_joiner_digit", + "codePoints": [ + "0031", + "2060", + "0032" + ], + "expect": "refuse", + "why": "U+2060 is Default_Ignorable" + }, + { + "name": "accept_abc", + "codePoints": [ + "0061", + "0062", + "0063" + ], + "expect": "accept", + "why": "letters" + }, + { + "name": "accept_digit_1", + "codePoints": [ + "0031" + ], + "expect": "accept", + "why": "a number" + }, + { + "name": "accept_padded_a_raw_compare", + "codePoints": [ + "0020", + "0061", + "0020" + ], + "expect": "accept", + "why": "raw compare: the id is echoed byte for byte, never trimmed, so the padding stays and the letter makes it valid" + }, + { + "name": "accept_e_acute_precomposed", + "codePoints": [ + "00E9" + ], + "expect": "accept", + "why": "a letter" + }, + { + "name": "accept_arabic_indic_digit_three", + "codePoints": [ + "0663" + ], + "expect": "accept", + "why": "category Nd, a number" + }, + { + "name": "accept_cjk_zhong", + "codePoints": [ + "4E2D" + ], + "expect": "accept", + "why": "a letter (Lo, not Default_Ignorable)" + }, + { + "name": "accept_letter_then_combining_acute", + "codePoints": [ + "0065", + "0301" + ], + "expect": "accept", + "why": "a combining mark after a letter is allowed" + }, + { + "name": "accept_256_ascii_letters", + "codePoints": [ + "0061" + ], + "expect": "accept", + "why": "256 bytes is the size limit and is allowed", + "repeat": 256 + } + ] + }, "schemaInvalid": [ { "name": "write_admitted_with_an_error_key", @@ -1182,7 +1456,8 @@ "wire": 7, "schemaInvalid": 7, "contentHashCases": 4, - "scenarioCases": 25 + "scenarioCases": 25, + "admittedMessageIdCases": 32 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index dc138452..150c10f8 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,6 +12,8 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. + **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. Why a new error version and not an extra key on v0: the v0 schema is closed (`additionalProperties: false`) and the TypeScript decoder (Ajv over the closed schemas) rejects unknown keys and unknown `schemaVersion`s, and `parseCultNetMessage` throws on an unknown version; Rust's decoder errors on an unknown version too. Unknown-key tolerance of the Rust, Python and Kotlin error decoders was not checked, so none is assumed, so a surprise key or frame breaks an old client's whole receive path. Old clients are therefore never sent either new frame. This reverses the "old clients ignore it" assumption, which is false for today's decoders. @@ -65,7 +67,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; the lone surrogate U+D800 is carried as a whole frame in `wireHex`, because it is not valid UTF-8 and only some runtimes' strings can hold it), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance status diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index e527692d..8955c52d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -48,6 +48,26 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // admittedMessageId: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, compared + // raw. The frame is built with this runtime's own encoder; a case with `wireHex` (the lone surrogate, invalid UTF-8) + // supplies the whole frame. + for (case in list(obj(vectors["admittedMessageId"])["cases"])) { + val c = obj(case) + check("admittedMessageId/${c["name"]}") { + val id = list(c["codePoints"]).joinToString("") { String(Character.toChars((it as String).toInt(16))) }.repeat(((c["repeat"] as Long?) ?: 1L).toInt()) + val bytes = (c["wireHex"] as String?)?.let(::unhex) ?: MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray() + val decoded = try { decodeReceiptFrame(parseCultNetMessage(bytes)) } catch (_: IOException) { null } + when { + c["expect"] == "refuse" -> if (decoded == null) emptyList() else listOf("accepted an id that must be refused (${c["why"]})") + decoded == null -> listOf("refused an id that must be accepted (${c["why"]})") + decoded.toWireMap()["messageId"] != id -> listOf("id not echoed raw: ${decoded.toWireMap()["messageId"]}") + else -> emptyList() + } + } + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { @@ -129,8 +149,9 @@ fun replayWriteReceiptVectors(path: String): Boolean { val loaded = mapOf( "wire" to list(vectors["wire"]).size, "schemaInvalid" to list(vectors["schemaInvalid"]).size, "contentHashCases" to list(obj(vectors["contentHash"])["cases"]).size, "scenarioCases" to list(obj(vectors["scenarios"])["cases"]).size, + "admittedMessageIdCases" to list(obj(vectors["admittedMessageId"])["cases"]).size, ) - val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/")) + val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/"), "admittedMessageIdCases" to ran("admittedMessageId/")) loaded.keys.filter { loaded[it] != (declared[it] as Long).toInt() || executed[it] != loaded[it] } .map { "$it: declared ${declared[it]}, loaded ${loaded[it]}, ran ${executed[it]}" } } diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 98e74287..0c42a6af 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -57,7 +57,7 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) @@ -68,6 +68,31 @@ class WriteReceiptVectorTests(unittest.TestCase): self.assertEqual(counts["schemaInvalid"], len(_VECTORS["schemaInvalid"])) self.assertEqual(counts["contentHashCases"], len(_VECTORS["contentHash"]["cases"])) self.assertEqual(counts["scenarioCases"], len(_VECTORS["scenarios"]["cases"])) + self.assertEqual(counts["admittedMessageIdCases"], len(_VECTORS["admittedMessageId"]["cases"])) + + def test_admitted_message_id_vectors_are_refused_or_accepted_raw(self) -> None: + # `admittedMessageId`: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, + # compared raw (never trimmed). A lone surrogate is not valid UTF-8, so its vector carries the whole frame. + cases = _VECTORS["admittedMessageId"]["cases"] + failures = [] + for case in cases: + raw = ( + bytes.fromhex(case["wireHex"]) + if "wireHex" in case + else _pack(_admitted(messageId="".join(chr(int(cp, 16)) for cp in case["codePoints"]) * case.get("repeat", 1))) + ) + try: + message = parse_message(raw) + except Exception: # noqa: BLE001 - any refusal counts + refused = True + else: + refused = False + wanted = "".join(chr(int(cp, 16)) for cp in case["codePoints"]) * case.get("repeat", 1) + if message.body["messageId"] != wanted: + failures.append(f"{case['name']}: id not echoed raw") + if refused != (case["expect"] == "refuse"): + failures.append(f"{case['name']}: expected {case['expect']}") + self.assertEqual(failures, []) def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 3d233a61..8dccf996 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -153,3 +153,51 @@ fn error_v0_refuses_the_v1_only_limit_reached_code() { as_v0[at + 15] = b'0'; assert!(decode_cultnet_message_from_slice(&as_v0, CultNetWireContract::CultNetSchemaV0).is_err()); } + +/// Vectors `admittedMessageId`: a write_admitted messageId needs a letter or number and no Default_Ignorable code +/// point, compared raw (never trimmed). The frame is the spec's `write_admitted_put` bytes with the id swapped in; a +/// case with `wireHex` (the lone surrogate, invalid UTF-8) supplies the whole frame. +#[test] +fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + let vectors: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = vectors["admittedMessageId"]["cases"].as_array().unwrap(); + assert_eq!(vectors["counts"]["admittedMessageIdCases"].as_u64().unwrap() as usize, cases.len()); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let mut failures = Vec::new(); + for case in cases { + let name = case["name"].as_str().unwrap(); + let id: String = case["codePoints"] + .as_array() + .unwrap() + .iter() + .filter_map(|cp| char::from_u32(u32::from_str_radix(cp.as_str().unwrap(), 16).unwrap())) + .collect::() + .repeat(case["repeat"].as_u64().unwrap_or(1) as usize); + let frame = match case["wireHex"].as_str() { + Some(hex) => unhex(hex), + None => { + let len = id.len(); + let header: Vec = if len < 32 { + vec![0xA0 | len as u8] + } else if len < 256 { + vec![0xD9, len as u8] + } else { + vec![0xDA, (len >> 8) as u8, len as u8] + }; + [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat() + } + }; + let decoded = decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0); + match (case["expect"].as_str().unwrap(), decoded) { + ("refuse", Ok(_)) => failures.push(format!("{name}: expected refuse")), + ("accept", Err(error)) => failures.push(format!("{name}: expected accept, got {error}")), + ("accept", Ok(CultNetMessage::WriteAdmitted { message_id, .. })) if message_id != id => { + failures.push(format!("{name}: id not echoed raw")) + } + _ => {} + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 6acb79cf..fbc8b6f0 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -280,3 +280,24 @@ test("a blank messageId, record key or tag is exactly Unicode White_Space, not t parseCultNetMessage(put({ tags: [named] })); } }); + +// `admittedMessageId` vectors: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, +// compared raw (never trimmed). JavaScript strings hold a lone surrogate, so the D800 vector is built from its code point. +test("write_admitted messageId: every admittedMessageId vector is refused or accepted raw", () => { + const vectorsPath = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); + assert.equal(vectors.counts.admittedMessageIdCases, vectors.admittedMessageId.cases.length); + const failures: string[] = []; + for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; expect: "refuse" | "accept" }[]) { + const id = vector.codePoints.map((cp) => String.fromCodePoint(parseInt(cp, 16))).join("").repeat(vector.repeat ?? 1); + let decodedId: string | undefined; + try { + decodedId = (parseCultNetMessage(admitted(id)) as CultNetWriteAdmittedMessage).messageId; + } catch { + decodedId = undefined; + } + if ((decodedId === undefined) !== (vector.expect === "refuse")) failures.push(`${vector.name}: expected ${vector.expect}`); + else if (decodedId !== undefined && decodedId !== id) failures.push(`${vector.name}: id not echoed raw`); + } + assert.deepEqual(failures, []); +}); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs new file mode 100644 index 00000000..4080f0d3 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // contracts/cultnet/interop/write-receipt-vectors.json `admittedMessageId`: a write_admitted messageId needs a letter or + // number and no Default_Ignorable code point, compared raw (never trimmed). The frame is built with this runtime's own + // encoder; the case with `wireHex` (a lone surrogate is invalid UTF-8) supplies the whole frame. + public sealed class CultNetWriteAdmittedMessageIdTests + { + private static JsonElement Vectors() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var path = Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + return JsonDocument.Parse(File.ReadAllText(path)).RootElement.Clone(); + } + + public static IEnumerable Cases() => + Vectors().GetProperty("admittedMessageId").GetProperty("cases").EnumerateArray().Select(c => c.GetProperty("name").GetString()!); + + [Test] + public void TheFileDeclaresHowManyCasesThisRunnerMustRun() => + Assert.That(Cases().Count(), Is.EqualTo(Vectors().GetProperty("counts").GetProperty("admittedMessageIdCases").GetInt32())); + + [TestCaseSource(nameof(Cases))] + public void AWriteAdmittedMessageIdIsRefusedOrAcceptedRaw(string name) + { + var vector = Vectors().GetProperty("admittedMessageId").GetProperty("cases").EnumerateArray() + .Single(c => c.GetProperty("name").GetString() == name); + var id = string.Concat(Enumerable.Repeat( + string.Concat(vector.GetProperty("codePoints").EnumerateArray() + .Select(cp => char.ConvertFromUtf32(Convert.ToInt32(cp.GetString(), 16)))), + vector.TryGetProperty("repeat", out var repeat) ? repeat.GetInt32() : 1)); + var bytes = vector.TryGetProperty("wireHex", out var wire) + ? Convert.FromHexString(wire.GetString()!) + : CultNetSchemaMessageSerialization.Serialize(new CultNetWriteAdmittedMessage + { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" }); + + CultNetWriteAdmittedMessage? decoded = null; + try { decoded = (CultNetWriteAdmittedMessage)CultNetSchemaMessageSerialization.Deserialize(bytes); } + catch (Exception) { /* a refusal */ } + + if (vector.GetProperty("expect").GetString() == "refuse") + Assert.That(decoded, Is.Null, vector.GetProperty("why").GetString()); + else + { + Assert.That(decoded, Is.Not.Null, vector.GetProperty("why").GetString()); + Assert.That(decoded!.MessageId, Is.EqualTo(id)); + } + } + } +} From 70f8135d2c713d33d203ecf0f6a7d41b2b59c674 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:41:11 +0000 Subject: [PATCH 17/40] vector: write_admitted messageId accepts a NUL between letters (rule: letter present, no Default_Ignorable) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt-vectors.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index a37d9660..7fa2e2fa 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -412,6 +412,16 @@ "expect": "accept", "why": "a combining mark after a letter is allowed" }, + { + "name": "accept_letter_nul_letter", + "codePoints": [ + "0061", + "0000", + "0062" + ], + "expect": "accept", + "why": "U+0000 is not Default_Ignorable and a letter is present, so the rule accepts; raw compare keeps the NUL" + }, { "name": "accept_256_ascii_letters", "codePoints": [ @@ -1457,7 +1467,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 32 + "admittedMessageIdCases": 33 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", From 4c1537885dd61f13628651ecf4633bc97fc10095 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:42:01 +0000 Subject: [PATCH 18/40] cultnet-ts: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare, no trim. The platform's regex property tables carry both properties; no table embedded. D800 vector now driven from wireHex. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 13 +++++++++++++ packages/cultnet-ts/test/write-receipt.test.ts | 8 +++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 3cc87dcd..a05a26df 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -950,6 +950,10 @@ export function parseCultNetMessage(input: unknown, wireContract: CultNetWireCon return input as CultNetDatabaseChangeRawMessage; } + if (schemaVersion === "cultnet.write_admitted.v0" && !isValidAdmittedMessageId((input as { messageId?: unknown }).messageId)) { + throw new Error("Validation failed for cultnet.write_admitted.v0: /messageId: must contain a letter or number and no Default_Ignorable code point"); + } + if (!validator(input)) { const keys = Object.keys(input as Record).join(","); throw new Error(`${renderValidationErrors(schemaVersion, validator)}; keys=${keys}`); @@ -1460,6 +1464,15 @@ function validateRawDocumentRecord( requireOptionalStringArray(candidate.tags, `${instancePath}/tags`); } +/** + * A write_admitted messageId is valid iff it holds a Unicode letter or number and no Default_Ignorable_Code_Point, compared raw + * (write-receipt.md; the id is echoed byte for byte, so nothing is trimmed). The platform's regex tables carry both properties, + * so this runtime embeds no table; the Kotlin, Python, C# and Rust decoders embed DerivedCoreProperties ranges. + */ +function isValidAdmittedMessageId(value: unknown): boolean { + return typeof value === "string" && /[\p{L}\p{N}]/u.test(value) && !/\p{Default_Ignorable_Code_Point}/u.test(value); +} + /** Blank is empty or only Unicode White_Space (write-receipt.md, rule 5), not `trim()`: JS trims U+FEFF and keeps U+0085. */ function isBlank(value: string): boolean { return /^\p{White_Space}*$/u.test(value); diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index fbc8b6f0..c447367a 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { decode } from "@msgpack/msgpack"; import { CULTNET_ERROR_CODES, CULTNET_ERROR_V1_CODES, @@ -282,17 +283,18 @@ test("a blank messageId, record key or tag is exactly Unicode White_Space, not t }); // `admittedMessageId` vectors: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, -// compared raw (never trimmed). JavaScript strings hold a lone surrogate, so the D800 vector is built from its code point. +// compared raw (never trimmed). The D800 vector is the frame's bytes (wireHex), decoded by the msgpack library like the others. test("write_admitted messageId: every admittedMessageId vector is refused or accepted raw", () => { const vectorsPath = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); assert.equal(vectors.counts.admittedMessageIdCases, vectors.admittedMessageId.cases.length); const failures: string[] = []; - for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; expect: "refuse" | "accept" }[]) { + for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; wireHex?: string; expect: "refuse" | "accept" }[]) { const id = vector.codePoints.map((cp) => String.fromCodePoint(parseInt(cp, 16))).join("").repeat(vector.repeat ?? 1); let decodedId: string | undefined; try { - decodedId = (parseCultNetMessage(admitted(id)) as CultNetWriteAdmittedMessage).messageId; + const frame = vector.wireHex ? decode(Uint8Array.from(Buffer.from(vector.wireHex, "hex"))) : admitted(id); + decodedId = (parseCultNetMessage(frame) as CultNetWriteAdmittedMessage).messageId; } catch { decodedId = undefined; } From bf55c10199d98bf9180b2ca3a041a36409c44c8a Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:42:44 +0000 Subject: [PATCH 19/40] vector: write_admitted messageId refuses U+FEFF before a letter (kills a trim-before-check decoder) Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt-vectors.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 7fa2e2fa..2958990b 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -351,6 +351,15 @@ "expect": "refuse", "why": "U+2060 is Default_Ignorable" }, + { + "name": "refuse_bom_before_letter", + "codePoints": [ + "FEFF", + "0061" + ], + "expect": "refuse", + "why": "U+FEFF is Default_Ignorable; JavaScript trim() would strip it and accept the rest, but the compare is raw" + }, { "name": "accept_abc", "codePoints": [ @@ -1467,7 +1476,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 33 + "admittedMessageIdCases": 34 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", From 688562c3164d2b68dfda4037fad25c6371092c74 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:43:26 +0000 Subject: [PATCH 20/40] cultnet-py: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare. unicodedata for L/N, an embedded Default_Ignorable range table (Unicode 16.0). Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet-py/src/cultnet_py/messages.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index f2d89296..8ece7f6e 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import unicodedata from dataclasses import dataclass from typing import Any @@ -69,6 +70,31 @@ def _is_name(value: Any) -> bool: return isinstance(value, str) and value != "" +# Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). This runtime has no +# platform table for the property, so it embeds it; the Kotlin, C# and Rust decoders embed the same ranges, TypeScript uses its +# regex engine's. Letters and numbers come from unicodedata (categories L* and N*); a code point the running Python's Unicode +# data does not assign (U+0378 in every version) is Cn, so it is refused either way. +_DEFAULT_IGNORABLE = ( + (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +) + + +def _is_admitted_message_id(value: Any) -> bool: + """write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never stripped).""" + if not isinstance(value, str): + return False + has_letter_or_number = False + for ch in value: + point = ord(ch) + if any(low <= point <= high for low, high in _DEFAULT_IGNORABLE): + return False + if unicodedata.category(ch)[0] in ("L", "N"): + has_letter_or_number = True + return has_letter_or_number + + def _validate_write_admitted_body(body: dict[str, Any]) -> None: """Refuse a cultnet.write_admitted.v0 body that is not exactly the five keys of its closed schema.""" expected = ("messageId", "operation", "schemaId", "recordKey") @@ -77,6 +103,8 @@ def _validate_write_admitted_body(body: dict[str, Any]) -> None: for key in ("messageId", "schemaId", "recordKey"): if not _is_name(body[key]): raise ValueError(f"cultnet.write_admitted.v0 {key} must be a non-empty string") + if not _is_admitted_message_id(body["messageId"]): + raise ValueError("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point") if body["operation"] not in ("put", "delete"): raise ValueError("cultnet.write_admitted.v0 operation must be put or delete") From d1a4ebdd6860defc13ebd7aa03fbf4ed5c2ff1fc Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:45:37 +0000 Subject: [PATCH 21/40] cultmesh-kotlin: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare. JDK Character categories for L/N, an embedded Default_Ignorable range table (Unicode 16.0). The MessagePack string reader now refuses invalid UTF-8 instead of substituting U+FFFD. Co-Authored-By: Claude Sonnet 5.5 --- .../kotlin/org/gamecult/cultmesh/CultMesh.kt | 5 +++- .../org/gamecult/cultmesh/WriteReceipt.kt | 27 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 4a06d7f5..2337448d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -19,6 +19,7 @@ import java.net.Socket import java.net.SocketTimeoutException import java.net.URI import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction import java.nio.charset.StandardCharsets import java.security.SecureRandom import java.security.MessageDigest @@ -6306,7 +6307,9 @@ class MessagePackReader(payload: ByteArray) { code == 0xdb -> input.readInt() else -> throw IOException("expected string") } - return String(readPayload(length), StandardCharsets.UTF_8) + // Strict: bytes that are not UTF-8 (a lone surrogate's ED A0 80) refuse the frame instead of becoming U+FFFD. + return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(readPayload(length))).toString() } private fun readBinaryAfterCode(code: Int): ByteArray { diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt index 4b38d942..523dfc2e 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -74,7 +74,32 @@ fun decodeWriteAdmitted(message: CultNetMessage): CultNetWriteAdmitted { closedKeys(message, setOf("messageId", "operation", "schemaId", "recordKey")) val operation = nonEmpty(message, "operation") if (operation != "put" && operation != "delete") throw IOException("operation must be put or delete") - return CultNetWriteAdmitted(nonEmpty(message, "messageId"), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) + return CultNetWriteAdmitted(admittedMessageId(message), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) +} + +// Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). The JDK has no such +// property, so this runtime embeds it; the Python, C# and Rust decoders embed the same ranges, TypeScript uses its regex +// engine's. Letters and numbers come from the JDK's Character categories (L*, and Nd, Nl, No); a code point the running JDK +// does not assign (U+0378 in every version) has no such category, so it is refused either way. +private val DEFAULT_IGNORABLE: List = listOf( + 0x00AD..0x00AD, 0x034F..0x034F, 0x061C..0x061C, 0x115F..0x1160, 0x17B4..0x17B5, 0x180B..0x180F, + 0x200B..0x200F, 0x202A..0x202E, 0x2060..0x206F, 0x3164..0x3164, 0xFE00..0xFE0F, 0xFEFF..0xFEFF, + 0xFFA0..0xFFA0, 0xFFF0..0xFFF8, 0x1BCA0..0x1BCA3, 0x1D173..0x1D17A, 0xE0000..0xE0FFF, +) + +/** write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never trimmed). */ +private fun admittedMessageId(message: CultNetMessage): String { + val id = nonEmpty(message, "messageId") + var hasLetterOrNumber = false + id.codePoints().forEach { cp -> + if (DEFAULT_IGNORABLE.any { cp in it }) throw IOException("messageId holds a Default_Ignorable code point") + when (Character.getType(cp).toByte()) { + Character.UPPERCASE_LETTER, Character.LOWERCASE_LETTER, Character.TITLECASE_LETTER, Character.MODIFIER_LETTER, + Character.OTHER_LETTER, Character.DECIMAL_DIGIT_NUMBER, Character.LETTER_NUMBER, Character.OTHER_NUMBER -> hasLetterOrNumber = true + } + } + if (!hasLetterOrNumber) throw IOException("messageId must contain a letter or number") + return id } /** Strict decode of error.v0 (no messageId, no limit_reached) or error.v1 (messageId required, string or nil, never empty). */ From d546ef9c7fb2dcdfc8ce1336a5ffd1277f333d66 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:49:50 +0000 Subject: [PATCH 22/40] GameCult.Networking: refuse a write_admitted messageId without a letter or number, with a Default_Ignorable code point, or with invalid UTF-8 Raw compare. .NET Unicode categories for L/N, an embedded Default_Ignorable range table (Unicode 16.0). Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 69 +++++++++++++++++++ .../CultNetWriteAdmittedMessageIdTests.cs | 2 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index cdcca912..9789e05e 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -1,4 +1,6 @@ using System; +using System.Buffers; +using System.Globalization; using MessagePack; namespace GameCult.Networking @@ -29,6 +31,7 @@ namespace GameCult.Networking public static T Deserialize(byte[] payload) where T : ICultNetSchemaMessage { if (payload == null) throw new ArgumentNullException(nameof(payload)); + if (typeof(T) == typeof(CultNetWriteAdmittedMessage)) CultNetAdmittedMessageId.Require(payload); return MessagePackSerializer.Deserialize(payload, Options); } @@ -98,4 +101,70 @@ namespace GameCult.Networking [Key("schemaVersion")] public string SchemaVersion { get; set; } = string.Empty; } } + + /// + /// The write_admitted messageId rule (write-receipt.md): a Unicode letter or number somewhere, no Default_Ignorable code + /// point anywhere, wire bytes that are valid UTF-8, compared raw (never trimmed). Letters and numbers come from the + /// runtime's Unicode categories (L*, Nd, Nl, No); a code point it does not assign (U+0378 in every version) has neither. + /// .NET has no Default_Ignorable property, so the ranges are embedded from DerivedCoreProperties.txt, Unicode 16.0 + /// (unchanged since 15.1); the Kotlin, Python and Rust decoders embed the same ranges, TypeScript uses its regex engine's. + /// + internal static class CultNetAdmittedMessageId + { + private static readonly (int Low, int High)[] DefaultIgnorable = + { + (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), + }; + + private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); + + /// Throws unless the frame's top-level messageId is a valid write_admitted id; other keys are left to the serializer. + public static void Require(byte[] payload) + { + var reader = new MessagePackReader(payload); + var entries = reader.ReadMapHeader(); + for (var index = 0; index < entries; index++) + { + var key = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; + if (key == "messageId" && reader.NextMessagePackType == MessagePackType.String) + { + string id; + try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } + catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } + if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); + return; + } + + if (key == null) reader.Skip(); + reader.Skip(); + } + } + + public static bool IsValid(string id) + { + var hasLetterOrNumber = false; + for (var at = 0; at < id.Length; at += char.IsSurrogatePair(id, at) ? 2 : 1) + { + var point = char.IsSurrogatePair(id, at) ? char.ConvertToUtf32(id, at) : id[at]; + if (Array.Exists(DefaultIgnorable, range => point >= range.Low && point <= range.High)) return false; + switch (CharUnicodeInfo.GetUnicodeCategory(point)) + { + case UnicodeCategory.UppercaseLetter: + case UnicodeCategory.LowercaseLetter: + case UnicodeCategory.TitlecaseLetter: + case UnicodeCategory.ModifierLetter: + case UnicodeCategory.OtherLetter: + case UnicodeCategory.DecimalDigitNumber: + case UnicodeCategory.LetterNumber: + case UnicodeCategory.OtherNumber: + hasLetterOrNumber = true; + break; + } + } + + return hasLetterOrNumber; + } + } } diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index 4080f0d3..f5cac528 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -35,7 +35,7 @@ namespace GameCult.Networking.Tests .Single(c => c.GetProperty("name").GetString() == name); var id = string.Concat(Enumerable.Repeat( string.Concat(vector.GetProperty("codePoints").EnumerateArray() - .Select(cp => char.ConvertFromUtf32(Convert.ToInt32(cp.GetString(), 16)))), + .Select(cp => Convert.ToInt32(cp.GetString(), 16)).Where(point => point < 0xD800 || point > 0xDFFF).Select(char.ConvertFromUtf32)), vector.TryGetProperty("repeat", out var repeat) ? repeat.GetInt32() : 1)); var bytes = vector.TryGetProperty("wireHex", out var wire) ? Convert.FromHexString(wire.GetString()!) From ba548d192df76bb533ec8bcdb119088693a1381e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 14:49:50 +0000 Subject: [PATCH 23/40] cultnet-rs: refuse a write_admitted messageId without a letter or number or with a Default_Ignorable code point Raw compare. std is_numeric for N; embedded letter (Unicode 15.1) and Default_Ignorable (16.0) range tables. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-rs/src/contracts.rs | 162 ++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 0ea2aa53..201632a2 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -680,7 +680,7 @@ fn validate_message(message: &CultNetMessage) -> Result<()> { } } CultNetMessage::WriteAdmitted { message_id, operation, schema_id, record_key } => { - require_non_empty(message_id, "messageId")?; + require_admitted_message_id(message_id)?; if operation != "put" && operation != "delete" { return Err(anyhow!("operation must be put or delete")); } @@ -934,6 +934,166 @@ fn require_non_empty(value: &str, field: &str) -> Result<()> { Ok(()) } +// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (\p{L}) or number (\p{N}) and no +// Default_Ignorable_Code_Point, compared raw (never trimmed). std has `char::is_numeric` (N) but no letter category +// (`is_alphabetic` is the Alphabetic property, which also takes combining marks), and no Default_Ignorable, so both tables +// are embedded: DerivedCoreProperties.txt ranges, Unicode 16.0 (unchanged since 15.1), and the general category L ranges of +// Unicode 15.1 (U+0378 is unassigned in every version, so it is refused either way). The Kotlin, Python and C# decoders embed +// the same Default_Ignorable ranges and take L/N from their platform; TypeScript uses its regex engine's tables. +// Invalid UTF-8 never reaches this check: the MessagePack str decode refuses it. +const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ + (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +]; + +const LETTERS: &[(u32, u32)] = &[ + (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB5, 0xB5), (0xBA, 0xBA), (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), + (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), + (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), + (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), + (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), (0x6E5, 0x6E6), (0x6EE, 0x6EF), (0x6FA, 0x6FC), (0x6FF, 0x6FF), + (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), (0x7CA, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), + (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), + (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), (0x950, 0x950), (0x958, 0x961), (0x971, 0x980), + (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), + (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), (0x9F0, 0x9F1), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), + (0xA13, 0xA28), (0xA2A, 0xA30), (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), + (0xA72, 0xA74), (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), + (0xABD, 0xABD), (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), + (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB71, 0xB71), + (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), (0xB9E, 0xB9F), + (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xC05, 0xC0C), (0xC0E, 0xC10), (0xC12, 0xC28), + (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), (0xC80, 0xC80), (0xC85, 0xC8C), + (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), + (0xCF1, 0xCF2), (0xD04, 0xD0C), (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), + (0xD5F, 0xD61), (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), + (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), (0xE8C, 0xEA3), + (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), (0xEDC, 0xEDF), + (0xF00, 0xF00), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), (0x1000, 0x102A), (0x103F, 0x103F), + (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), (0x106E, 0x1070), (0x1075, 0x1081), + (0x108E, 0x108E), (0x10A0, 0x10C5), (0x10C7, 0x10C7), (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), + (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), + (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), + (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), + (0x1401, 0x166C), (0x166F, 0x167F), (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16F1, 0x16F8), (0x1700, 0x1711), + (0x171F, 0x1731), (0x1740, 0x1751), (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), + (0x17DC, 0x17DC), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), (0x18B0, 0x18F5), + (0x1900, 0x191E), (0x1950, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), (0x1A00, 0x1A16), + (0x1A20, 0x1A54), (0x1AA7, 0x1AA7), (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B83, 0x1BA0), (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), (0x1C00, 0x1C23), (0x1C4D, 0x1C4F), (0x1C5A, 0x1C7D), (0x1C80, 0x1C88), (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), (0x1F18, 0x1F1D), (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), (0x2071, 0x2071), (0x207F, 0x207F), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), + (0x210A, 0x2113), (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), + (0x212A, 0x212D), (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2183, 0x2184), + (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), + (0x3005, 0x3006), (0x3031, 0x3035), (0x303B, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), + (0x30FC, 0x30FF), (0x3105, 0x312F), (0x3131, 0x318E), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3400, 0x4DBF), + (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), (0xA500, 0xA60C), (0xA610, 0xA61F), (0xA62A, 0xA62B), (0xA640, 0xA66E), + (0xA67F, 0xA69D), (0xA6A0, 0xA6E5), (0xA717, 0xA71F), (0xA722, 0xA788), (0xA78B, 0xA7CA), (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), (0xA7D5, 0xA7D9), (0xA7F2, 0xA801), (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), + (0xA840, 0xA873), (0xA882, 0xA8B3), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA90A, 0xA925), + (0xA930, 0xA946), (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9CF), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), (0xAA00, 0xAA28), (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), + (0xAB20, 0xAB26), (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), + (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), (0x10028, 0x1003A), + (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), (0x10280, 0x1029C), + (0x102A0, 0x102D0), (0x10300, 0x1031F), (0x1032D, 0x10340), (0x10342, 0x10349), (0x10350, 0x10375), + (0x10380, 0x1039D), (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x10400, 0x1049D), (0x104B0, 0x104D3), + (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), (0x1057C, 0x1058A), + (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), (0x105B3, 0x105B9), + (0x105BB, 0x105BC), (0x10600, 0x10736), (0x10740, 0x10755), (0x10760, 0x10767), (0x10780, 0x10785), + (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), (0x10808, 0x10808), (0x1080A, 0x10835), + (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), (0x10860, 0x10876), (0x10880, 0x1089E), + (0x108E0, 0x108F2), (0x108F4, 0x108F5), (0x10900, 0x10915), (0x10920, 0x10939), (0x10980, 0x109B7), + (0x109BE, 0x109BF), (0x10A00, 0x10A00), (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), (0x10A80, 0x10A9C), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10B00, 0x10B35), + (0x10B40, 0x10B55), (0x10B60, 0x10B72), (0x10B80, 0x10B91), (0x10C00, 0x10C48), (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), (0x10D00, 0x10D23), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), (0x10F30, 0x10F45), (0x10F70, 0x10F81), (0x10FB0, 0x10FC4), (0x10FE0, 0x10FF6), + (0x11003, 0x11037), (0x11071, 0x11072), (0x11075, 0x11075), (0x11083, 0x110AF), (0x110D0, 0x110E8), + (0x11103, 0x11126), (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), + (0x11183, 0x111B2), (0x111C1, 0x111C4), (0x111DA, 0x111DA), (0x111DC, 0x111DC), (0x11200, 0x11211), + (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), + (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x11305, 0x1130C), (0x1130F, 0x11310), + (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), (0x1133D, 0x1133D), + (0x11350, 0x11350), (0x1135D, 0x11361), (0x11400, 0x11434), (0x11447, 0x1144A), (0x1145F, 0x11461), + (0x11480, 0x114AF), (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x11580, 0x115AE), (0x115D8, 0x115DB), + (0x11600, 0x1162F), (0x11644, 0x11644), (0x11680, 0x116AA), (0x116B8, 0x116B8), (0x11700, 0x1171A), + (0x11740, 0x11746), (0x11800, 0x1182B), (0x118A0, 0x118DF), (0x118FF, 0x11906), (0x11909, 0x11909), + (0x1190C, 0x11913), (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), + (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), (0x11C40, 0x11C40), (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), (0x11D08, 0x11D09), (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D60, 0x11D65), + (0x11D67, 0x11D68), (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), + (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11FB0, 0x11FB0), (0x12000, 0x12399), (0x12480, 0x12543), + (0x12F90, 0x12FF0), (0x13000, 0x1342F), (0x13441, 0x13446), (0x14400, 0x14646), (0x16800, 0x16A38), + (0x16A40, 0x16A5E), (0x16A70, 0x16ABE), (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), + (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16E40, 0x16E7F), (0x16F00, 0x16F4A), (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), (0x17000, 0x187F7), (0x18800, 0x18CD5), + (0x18D00, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), + (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), (0x1BC90, 0x1BC99), (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), (0x1D540, 0x1D544), + (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), (0x1E137, 0x1E13D), (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), (0x1E4D0, 0x1E4EB), (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x20000, 0x2A6DF), (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x323AF), +]; + +fn in_ranges(ranges: &[(u32, u32)], point: u32) -> bool { + ranges.binary_search_by(|&(low, high)| { + if point < low { + std::cmp::Ordering::Greater + } else if point > high { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} + +fn require_admitted_message_id(value: &str) -> Result<()> { + let mut has_letter_or_number = false; + for ch in value.chars() { + let point = ch as u32; + if in_ranges(DEFAULT_IGNORABLE, point) { + return Err(anyhow!("CultNet field messageId must not hold a Default_Ignorable code point")); + } + has_letter_or_number |= ch.is_numeric() || in_ranges(LETTERS, point); + } + if !has_letter_or_number { + return Err(anyhow!("CultNet field messageId must contain a letter or number")); + } + Ok(()) +} + fn require_exact(value: &str, expected: &str, field: &str) -> Result<()> { if value != expected { return Err(anyhow!("{field} must be {expected}")); From cc931932fc0bf32adaecac74345db84edf4314e1 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:00:54 +0000 Subject: [PATCH 24/40] tests first: Unicode 16.0 pin vectors (failing) Failing before the fix: Rust 1 (16100), Python 2 (16100, 10D40), Kotlin 4 (16100, 10D40, 2EBF0, 2EE5D). TypeScript and C# already pass: their platform data is 16.0 or later. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 2958990b..c7f47738 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -150,7 +150,7 @@ ] }, "admittedMessageId": { - "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Scope: write_admitted only; error.v1 is untouched.", + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Letters, numbers and Default_Ignorable are those of Unicode 16.0 in every runtime, from embedded tables (no platform category lookup), so a runtime's own Unicode version cannot change a result. Scope: write_admitted only; error.v1 is untouched.", "cases": [ { "name": "refuse_braille_blank_2800", @@ -360,6 +360,54 @@ "expect": "refuse", "why": "U+FEFF is Default_Ignorable; JavaScript trim() would strip it and accept the rest, but the compare is raw" }, + { + "name": "accept_unicode16_gurung_khema_letter_16100", + "codePoints": [ + "16100" + ], + "expect": "accept", + "why": "U+16100 GURUNG KHEMA LETTER A is Lo and new in Unicode 16.0 (unassigned in 15.1); the rule is pinned to 16.0 in every runtime" + }, + { + "name": "accept_unicode16_garay_digit_zero_10D40", + "codePoints": [ + "10D40" + ], + "expect": "accept", + "why": "U+10D40 GARAY DIGIT ZERO is Nd and new in Unicode 16.0" + }, + { + "name": "accept_unicode15_1_cjk_ext_i_2EBF0", + "codePoints": [ + "2EBF0" + ], + "expect": "accept", + "why": "U+2EBF0, first of CJK Unified Ideographs Extension I, is Lo and new in Unicode 15.1" + }, + { + "name": "accept_last_cjk_ext_i_2EE5D", + "codePoints": [ + "2EE5D" + ], + "expect": "accept", + "why": "U+2EE5D is the last assigned code point of Extension I (Lo)" + }, + { + "name": "refuse_after_cjk_ext_i_2EE5E", + "codePoints": [ + "2EE5E" + ], + "expect": "refuse", + "why": "U+2EE5E is unassigned in Unicode 16.0, so neither a letter nor a number" + }, + { + "name": "refuse_unassigned_in_16_0_10D90", + "codePoints": [ + "10D90" + ], + "expect": "refuse", + "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0" + }, { "name": "accept_abc", "codePoints": [ @@ -1476,7 +1524,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 34 + "admittedMessageIdCases": 40 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", From 3d37fda49638c4171ab654b1a72388aaee175449 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 25/40] write_admitted messageId: pin Unicode 16.0, one generator for the embedded tables scripts/gen-unicode-messageid-tables.py emits each runtime's Letter-or-Number and Default_Ignorable tables from the two UCD 16.0 files; --check fails on a stale or hand-edited table. Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- scripts/gen-unicode-messageid-tables.py | 111 +++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 scripts/gen-unicode-messageid-tables.py diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 150c10f8..10f42a6a 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. diff --git a/scripts/gen-unicode-messageid-tables.py b/scripts/gen-unicode-messageid-tables.py new file mode 100644 index 00000000..96d1a205 --- /dev/null +++ b/scripts/gen-unicode-messageid-tables.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Generate the Unicode 16.0 tables of the write_admitted messageId rule for the five CultNet runtimes. + +The rule (contracts/cultnet/interop/write-receipt.md): a messageId is valid iff it holds a letter (Lu Ll Lt Lm Lo) or a number +(Nd Nl No) and no Default_Ignorable_Code_Point, all as of Unicode 16.0. The tables are embedded in each runtime so that no +platform's own Unicode version can change a result. Bumping the version is one coordinated change: run this script on the new +UCD files, commit the five regenerated files and the vectors that pin the new version, together. + + gen-unicode-messageid-tables.py DerivedGeneralCategory.txt DerivedCoreProperties.txt [--check] + +The inputs are https://www.unicode.org/Public/16.0.0/ucd/extracted/DerivedGeneralCategory.txt and +https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --check nothing is written; the exit status is 1 if +any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only. +""" +import hashlib +import re +import sys +from pathlib import Path + +VERSION = "16.0.0" +ROOT = Path(__file__).resolve().parent.parent +LETTER_OR_NUMBER_CATEGORIES = {"Lu", "Ll", "Lt", "Lm", "Lo", "Nd", "Nl", "No"} + +TARGETS = { + "ts": ROOT / "packages/cultnet-ts/src/unicode-messageid-tables.ts", + "py": ROOT / "packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py", + "kt": ROOT / "packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt", + "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs", + "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs", +} + + +def parse(path, wanted): + """Ranges (low, high) of the lines `LOW[..HIGH] ; VALUE`, VALUE accepted by [wanted], merged and sorted.""" + points = [] + for line in Path(path).read_text(encoding="utf-8").splitlines(): + m = re.match(r"([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*(\S+)", line) + if m and wanted(m.group(3)): + points.append((int(m.group(1), 16), int(m.group(2) or m.group(1), 16))) + points.sort() + merged = [] + for low, high in points: + if merged and low <= merged[-1][1] + 1: + merged[-1] = (merged[-1][0], max(merged[-1][1], high)) + else: + merged.append((low, high)) + return merged + + +def flat(ranges): + return [value for pair in ranges for value in pair] + + +def wrapped(items, indent, render, width=118): + lines, current = [], indent + for item in items: + text = render(item) + ", " + if len(current) + len(text) > width: + lines.append(current.rstrip()) + current = indent + current += text + lines.append(current.rstrip()) + return "\n".join(lines) + + +def emit(target, letters, ignorable, header): + num = lambda v: f"0x{v:X}" + if target == "ts": + body = lambda name, r: f"export const {name}: readonly number[] = [\n{wrapped(flat(r), ' ', num)}\n];\n" + return f"// {header}\n\n// [low, high] pairs, sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "py": + body = lambda name, r: f"{name}: tuple[int, ...] = (\n{wrapped(flat(r), ' ', num)}\n)\n" + return f"# {header}\n\n# (low, high) pairs, flattened and sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "kt": + body = lambda name, r: f"internal val {name}: IntArray = intArrayOf(\n{wrapped(flat(r), ' ', num)}\n)\n" + return f"// {header}\n\npackage org.gamecult.cultmesh\n\n// (low, high) pairs, flattened and sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "cs": + body = lambda name, r: f" public static readonly int[] {name} =\n {{\n{wrapped(flat(r), ' ', num)}\n }};\n" + return f"// {header}\n\nnamespace GameCult.Networking\n{{\n internal static class CultNetUnicodeMessageIdTables\n {{\n // (low, high) pairs, flattened and sorted.\n{body('LetterOrNumber', letters)}\n{body('DefaultIgnorable', ignorable)} }}\n}}\n" + body = lambda name, r: f"pub const {name}: &[(u32, u32)] = &[\n{wrapped(r, ' ', lambda p: f'({num(p[0])}, {num(p[1])})')}\n];\n" + return f"// {header}\n\n// (low, high) pairs, sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + + +def main(): + args = [a for a in sys.argv[1:] if a != "--check"] + check = "--check" in sys.argv[1:] + if len(args) != 2: + sys.exit(__doc__) + gc, dcp = args + for path, marker in ((gc, f"DerivedGeneralCategory-{VERSION}"), (dcp, f"DerivedCoreProperties-{VERSION}")): + if marker not in Path(path).read_text(encoding="utf-8").splitlines()[0]: + sys.exit(f"{path} is not {marker}.txt") + letters = parse(gc, lambda v: v in LETTER_OR_NUMBER_CATEGORIES) + ignorable = parse(dcp, lambda v: v == "Default_Ignorable_Code_Point") + digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16] + header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit" + stale = [] + for target, path in TARGETS.items(): + text = emit(target, letters, ignorable, header) + if check: + if not path.exists() or path.read_text(encoding="utf-8") != text: + stale.append(str(path.relative_to(ROOT))) + else: + path.write_text(text, encoding="utf-8") + if stale: + sys.exit("stale or hand-edited: " + ", ".join(stale)) + print(f"{'checked' if check else 'wrote'} {len(TARGETS)} tables: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") + + +if __name__ == "__main__": + main() From de25b50abdb4ac90b0512d7436aa59605ad9c678 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 26/40] cultnet-ts: write_admitted messageId from embedded Unicode 16.0 tables, not the regex engine's Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 38 +++++- .../src/unicode-messageid-tables.ts | 126 ++++++++++++++++++ 2 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 packages/cultnet-ts/src/unicode-messageid-tables.ts diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index a05a26df..09d5d823 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -10,6 +10,7 @@ import verifySchema from "../contracts/cultnet/cultnet.verify.schema.json"; import loginSuccessSchema from "../contracts/cultnet/cultnet.login-success.schema.json"; import errorSchema from "../contracts/cultnet/cultnet.error.schema.json"; import errorV1Schema from "../contracts/cultnet/interop/cultnet.error.v1.schema.json"; +import { DEFAULT_IGNORABLE, LETTER_OR_NUMBER } from "./unicode-messageid-tables"; import writeAdmittedSchema from "../contracts/cultnet/interop/cultnet.write-admitted.schema.json"; import sampleChangeNameSchema from "../contracts/cultnet/cultnet.sample-change-name.schema.json"; import sampleChatSchema from "../contracts/cultnet/cultnet.sample-chat.schema.json"; @@ -1465,12 +1466,41 @@ function validateRawDocumentRecord( } /** - * A write_admitted messageId is valid iff it holds a Unicode letter or number and no Default_Ignorable_Code_Point, compared raw - * (write-receipt.md; the id is echoed byte for byte, so nothing is trimmed). The platform's regex tables carry both properties, - * so this runtime embeds no table; the Kotlin, Python, C# and Rust decoders embed DerivedCoreProperties ranges. + * A write_admitted messageId is valid iff it holds a Unicode letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) and no + * Default_Ignorable_Code_Point, as of Unicode 16.0, compared raw (write-receipt.md; the id is echoed byte for byte, so nothing + * is trimmed). The tables are embedded (unicode-messageid-tables.ts, generated by scripts/gen-unicode-messageid-tables.py), not + * the regex engine's, so the running Node's Unicode version changes no result; the other four runtimes embed the same tables. */ function isValidAdmittedMessageId(value: unknown): boolean { - return typeof value === "string" && /[\p{L}\p{N}]/u.test(value) && !/\p{Default_Ignorable_Code_Point}/u.test(value); + if (typeof value !== "string") { + return false; + } + let hasLetterOrNumber = false; + for (const ch of value) { + const point = ch.codePointAt(0) as number; + if (inRanges(DEFAULT_IGNORABLE, point)) { + return false; + } + hasLetterOrNumber = hasLetterOrNumber || inRanges(LETTER_OR_NUMBER, point); + } + return hasLetterOrNumber; +} + +/** Whether [point] lies in the flattened, sorted [low, high] pairs. */ +function inRanges(ranges: readonly number[], point: number): boolean { + let low = 0; + let high = ranges.length / 2 - 1; + while (low <= high) { + const mid = (low + high) >> 1; + if (point < ranges[2 * mid]) { + high = mid - 1; + } else if (point > ranges[2 * mid + 1]) { + low = mid + 1; + } else { + return true; + } + } + return false; } /** Blank is empty or only Unicode White_Space (write-receipt.md, rule 5), not `trim()`: JS trims U+FEFF and keeps U+0085. */ diff --git a/packages/cultnet-ts/src/unicode-messageid-tables.ts b/packages/cultnet-ts/src/unicode-messageid-tables.ts new file mode 100644 index 00000000..3374a011 --- /dev/null +++ b/packages/cultnet-ts/src/unicode-messageid-tables.ts @@ -0,0 +1,126 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +// [low, high] pairs, sorted. +export const LETTER_OR_NUMBER: readonly number[] = [ + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, 0x103F, + 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, + 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, 0x124A, + 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, 0x12B2, + 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, 0x1318, + 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, 0x1681, + 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, 0x176E, + 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, 0x1820, + 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, 0x1970, + 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, 0x1A90, + 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1C00, + 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, 0x1CEE, + 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, + 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, 0x1FB6, + 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, 0x1FF2, + 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, 0x2107, + 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, + 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, 0x24EA, + 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, 0x2D27, + 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, 0x2DB0, + 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, 0x3005, + 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, 0x30FC, + 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, 0x3248, + 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, 0xA500, + 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, 0xA78B, + 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, + 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, 0xA8FD, + 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, 0xA9E6, + 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7E, + 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, 0xAAE0, + 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, 0xAB30, + 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0xF900, + 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, 0xFB38, + 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, 0xFD92, + 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, 0xFF66, + 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, 0x10026, + 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, 0x10133, + 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, 0x10323, + 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, 0x103D5, + 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, 0x10563, + 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, 0x105B1, + 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, + 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, + 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, 0x109CF, + 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, 0x10A7E, + 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, 0x10B55, + 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, + 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, 0x10EA9, + 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, 0x10F81, + 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, 0x11075, + 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, 0x11144, + 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, 0x111DA, + 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, 0x11286, + 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, 0x112F9, + 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, 0x11339, + 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, 0x1138E, + 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, 0x1144A, + 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, 0x114D9, + 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, 0x116AA, + 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, 0x11746, + 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, 0x11916, + 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D0, + 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, 0x11A50, + 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, + 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, 0x11D09, + 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, 0x11D89, + 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, 0x11F33, + 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, 0x12543, + 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, 0x1611D, + 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, 0x16AC9, + 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, + 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, 0x16F50, + 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, 0x18D08, + 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, 0x1B152, + 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, 0x1BC88, + 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, 0x1D4B9, + 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, 0x1D51C, + 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, 0x1D6A5, + 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, 0x1D74E, + 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, 0x1D7FF, + 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, 0x1E149, + 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, 0x1E4F9, + 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, 0x1E7FE, + 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, 0x1ECAB, + 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, + 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, 0x1EE39, + 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, 0x1EE4F, + 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, 0x1EE5D, + 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, 0x1EE77, + 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, 0x1EEA9, + 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, 0x2B81D, + 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, 0x323AF, +]; + +export const DEFAULT_IGNORABLE: readonly number[] = [ + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +]; From 87f53498ed1db7709ade57241bc9b47bf05619e0 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 27/40] cultnet-py: write_admitted messageId from embedded Unicode 16.0 tables, not unicodedata Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet-py/src/cultnet_py/messages.py | 34 +++-- .../cultnet_py/unicode_messageid_tables.py | 127 ++++++++++++++++++ 2 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index 8ece7f6e..d5e26b7a 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -1,12 +1,13 @@ from __future__ import annotations import hashlib -import unicodedata from dataclasses import dataclass from typing import Any import msgpack +from .unicode_messageid_tables import DEFAULT_IGNORABLE, LETTER_OR_NUMBER + @dataclass(frozen=True) class CultNetMessage: @@ -70,28 +71,33 @@ def _is_name(value: Any) -> bool: return isinstance(value, str) and value != "" -# Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). This runtime has no -# platform table for the property, so it embeds it; the Kotlin, C# and Rust decoders embed the same ranges, TypeScript uses its -# regex engine's. Letters and numbers come from unicodedata (categories L* and N*); a code point the running Python's Unicode -# data does not assign (U+0378 in every version) is Cn, so it is refused either way. -_DEFAULT_IGNORABLE = ( - (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), - (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), - (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), -) +# write-receipt.md: a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no Default_Ignorable code point anywhere, as of +# Unicode 16.0, compared raw (never stripped). The tables are embedded (unicode_messageid_tables.py, generated by +# scripts/gen-unicode-messageid-tables.py), not unicodedata's, so the running Python's Unicode version changes no result; +# the other four runtimes embed the same generated tables. +def _in_ranges(ranges: tuple[int, ...], point: int) -> bool: + """Whether [point] lies in the flattened, sorted (low, high) pairs.""" + low, high = 0, len(ranges) // 2 - 1 + while low <= high: + mid = (low + high) // 2 + if point < ranges[2 * mid]: + high = mid - 1 + elif point > ranges[2 * mid + 1]: + low = mid + 1 + else: + return True + return False def _is_admitted_message_id(value: Any) -> bool: - """write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never stripped).""" if not isinstance(value, str): return False has_letter_or_number = False for ch in value: point = ord(ch) - if any(low <= point <= high for low, high in _DEFAULT_IGNORABLE): + if _in_ranges(DEFAULT_IGNORABLE, point): return False - if unicodedata.category(ch)[0] in ("L", "N"): - has_letter_or_number = True + has_letter_or_number = has_letter_or_number or _in_ranges(LETTER_OR_NUMBER, point) return has_letter_or_number diff --git a/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py b/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py new file mode 100644 index 00000000..163e3c08 --- /dev/null +++ b/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py @@ -0,0 +1,127 @@ +# generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +# (low, high) pairs, flattened and sorted. +LETTER_OR_NUMBER: tuple[int, ...] = ( + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, + 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, + 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, + 0x124A, 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, + 0x12B2, 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, + 0x1318, 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, + 0x1681, 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, + 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, + 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, + 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, + 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, + 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, + 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, + 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, + 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, + 0x2107, 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, + 0x212A, 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, + 0x24EA, 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, + 0x2D27, 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, + 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, + 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, + 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, + 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, + 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, + 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, + 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, + 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, + 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, + 0xAAE0, 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB30, 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, + 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, + 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, + 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, + 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, + 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, + 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, + 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, + 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, + 0x10767, 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, + 0x10835, 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, + 0x108AF, 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, + 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, + 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, + 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, + 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, + 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, + 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, + 0x11144, 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, + 0x111DA, 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, + 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, + 0x112F9, 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, + 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, + 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, + 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, + 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, + 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, + 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, + 0x119D0, 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, + 0x11A50, 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, + 0x11C08, 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, + 0x11D09, 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, + 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, + 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, + 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, + 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, + 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, + 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, + 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, + 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, + 0x1BC88, 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, + 0x1D454, 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, + 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, + 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, + 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, + 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, + 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, + 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, + 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, + 0x1ECAB, 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, + 0x1EE1F, 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, + 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, + 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, + 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, + 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, + 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, + 0x323AF, +) + +DEFAULT_IGNORABLE: tuple[int, ...] = ( + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +) From c4f6e768395e2467346320c7ecff270791265ce5 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:04:59 +0000 Subject: [PATCH 28/40] cultmesh-kotlin: write_admitted messageId from embedded Unicode 16.0 tables, not the JDK's Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh/UnicodeMessageIdTables.kt | 129 ++++++++++++++++++ .../org/gamecult/cultmesh/WriteReceipt.kt | 34 +++-- 2 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt new file mode 100644 index 00000000..7bc4bdc8 --- /dev/null +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt @@ -0,0 +1,129 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +package org.gamecult.cultmesh + +// (low, high) pairs, flattened and sorted. +internal val LETTER_OR_NUMBER: IntArray = intArrayOf( + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, + 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, + 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, + 0x124A, 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, + 0x12B2, 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, + 0x1318, 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, + 0x1681, 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, + 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, + 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, + 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, + 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, + 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, + 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, + 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, + 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, + 0x2107, 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, + 0x212A, 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, + 0x24EA, 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, + 0x2D27, 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, + 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, + 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, + 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, + 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, + 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, + 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, + 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, + 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, + 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, + 0xAAE0, 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB30, 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, + 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, + 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, + 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, + 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, + 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, + 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, + 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, + 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, + 0x10767, 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, + 0x10835, 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, + 0x108AF, 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, + 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, + 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, + 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, + 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, + 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, + 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, + 0x11144, 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, + 0x111DA, 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, + 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, + 0x112F9, 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, + 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, + 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, + 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, + 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, + 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, + 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, + 0x119D0, 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, + 0x11A50, 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, + 0x11C08, 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, + 0x11D09, 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, + 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, + 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, + 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, + 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, + 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, + 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, + 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, + 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, + 0x1BC88, 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, + 0x1D454, 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, + 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, + 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, + 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, + 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, + 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, + 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, + 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, + 0x1ECAB, 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, + 0x1EE1F, 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, + 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, + 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, + 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, + 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, + 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, + 0x323AF, +) + +internal val DEFAULT_IGNORABLE: IntArray = intArrayOf( + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt index 523dfc2e..9cf9c402 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -77,26 +77,30 @@ fun decodeWriteAdmitted(message: CultNetMessage): CultNetWriteAdmitted { return CultNetWriteAdmitted(admittedMessageId(message), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) } -// Default_Ignorable_Code_Point ranges, DerivedCoreProperties.txt, Unicode 16.0 (unchanged since 15.1). The JDK has no such -// property, so this runtime embeds it; the Python, C# and Rust decoders embed the same ranges, TypeScript uses its regex -// engine's. Letters and numbers come from the JDK's Character categories (L*, and Nd, Nl, No); a code point the running JDK -// does not assign (U+0378 in every version) has no such category, so it is refused either way. -private val DEFAULT_IGNORABLE: List = listOf( - 0x00AD..0x00AD, 0x034F..0x034F, 0x061C..0x061C, 0x115F..0x1160, 0x17B4..0x17B5, 0x180B..0x180F, - 0x200B..0x200F, 0x202A..0x202E, 0x2060..0x206F, 0x3164..0x3164, 0xFE00..0xFE0F, 0xFEFF..0xFEFF, - 0xFFA0..0xFFA0, 0xFFF0..0xFFF8, 0x1BCA0..0x1BCA3, 0x1D173..0x1D17A, 0xE0000..0xE0FFF, -) +// write-receipt.md: a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no Default_Ignorable code point anywhere, as of +// Unicode 16.0, compared raw (never trimmed). The tables are embedded (UnicodeMessageIdTables.kt, generated by +// scripts/gen-unicode-messageid-tables.py), not the JDK's, so the running JVM's Unicode version changes no result; the other +// four runtimes embed the same generated tables. +private fun inRanges(ranges: IntArray, point: Int): Boolean { + var low = 0 + var high = ranges.size / 2 - 1 + while (low <= high) { + val mid = (low + high) ushr 1 + when { + point < ranges[2 * mid] -> high = mid - 1 + point > ranges[2 * mid + 1] -> low = mid + 1 + else -> return true + } + } + return false +} -/** write-receipt.md: a letter or number somewhere, no Default_Ignorable code point anywhere, compared raw (never trimmed). */ private fun admittedMessageId(message: CultNetMessage): String { val id = nonEmpty(message, "messageId") var hasLetterOrNumber = false id.codePoints().forEach { cp -> - if (DEFAULT_IGNORABLE.any { cp in it }) throw IOException("messageId holds a Default_Ignorable code point") - when (Character.getType(cp).toByte()) { - Character.UPPERCASE_LETTER, Character.LOWERCASE_LETTER, Character.TITLECASE_LETTER, Character.MODIFIER_LETTER, - Character.OTHER_LETTER, Character.DECIMAL_DIGIT_NUMBER, Character.LETTER_NUMBER, Character.OTHER_NUMBER -> hasLetterOrNumber = true - } + if (inRanges(DEFAULT_IGNORABLE, cp)) throw IOException("messageId holds a Default_Ignorable code point") + if (inRanges(LETTER_OR_NUMBER, cp)) hasLetterOrNumber = true } if (!hasLetterOrNumber) throw IOException("messageId must contain a letter or number") return id From c2d2c1d126a719bd2ff9a38929350dfff31a249e Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:05:00 +0000 Subject: [PATCH 29/40] GameCult.Networking: write_admitted messageId from embedded Unicode 16.0 tables, not .NET's Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 47 +++--- .../CultNetUnicodeMessageIdTables.cs | 144 ++++++++++++++++++ 2 files changed, 164 insertions(+), 27 deletions(-) create mode 100644 src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index 9789e05e..3069384a 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -1,6 +1,5 @@ using System; using System.Buffers; -using System.Globalization; using MessagePack; namespace GameCult.Networking @@ -103,21 +102,13 @@ namespace GameCult.Networking } /// - /// The write_admitted messageId rule (write-receipt.md): a Unicode letter or number somewhere, no Default_Ignorable code - /// point anywhere, wire bytes that are valid UTF-8, compared raw (never trimmed). Letters and numbers come from the - /// runtime's Unicode categories (L*, Nd, Nl, No); a code point it does not assign (U+0378 in every version) has neither. - /// .NET has no Default_Ignorable property, so the ranges are embedded from DerivedCoreProperties.txt, Unicode 16.0 - /// (unchanged since 15.1); the Kotlin, Python and Rust decoders embed the same ranges, TypeScript uses its regex engine's. + /// The write_admitted messageId rule (write-receipt.md): a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no + /// Default_Ignorable code point anywhere, as of Unicode 16.0, wire bytes that are valid UTF-8, compared raw (never trimmed). + /// The tables are embedded (CultNetUnicodeMessageIdTables.cs, generated by scripts/gen-unicode-messageid-tables.py), not + /// .NET's, so the running runtime's Unicode version changes no result; the other four runtimes embed the same tables. /// internal static class CultNetAdmittedMessageId { - private static readonly (int Low, int High)[] DefaultIgnorable = - { - (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), - (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), - (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), - }; - private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); /// Throws unless the frame's top-level messageId is a valid write_admitted id; other keys are left to the serializer. @@ -148,23 +139,25 @@ namespace GameCult.Networking for (var at = 0; at < id.Length; at += char.IsSurrogatePair(id, at) ? 2 : 1) { var point = char.IsSurrogatePair(id, at) ? char.ConvertToUtf32(id, at) : id[at]; - if (Array.Exists(DefaultIgnorable, range => point >= range.Low && point <= range.High)) return false; - switch (CharUnicodeInfo.GetUnicodeCategory(point)) - { - case UnicodeCategory.UppercaseLetter: - case UnicodeCategory.LowercaseLetter: - case UnicodeCategory.TitlecaseLetter: - case UnicodeCategory.ModifierLetter: - case UnicodeCategory.OtherLetter: - case UnicodeCategory.DecimalDigitNumber: - case UnicodeCategory.LetterNumber: - case UnicodeCategory.OtherNumber: - hasLetterOrNumber = true; - break; - } + if (InRanges(CultNetUnicodeMessageIdTables.DefaultIgnorable, point)) return false; + hasLetterOrNumber |= InRanges(CultNetUnicodeMessageIdTables.LetterOrNumber, point); } return hasLetterOrNumber; } + + private static bool InRanges(int[] ranges, int point) + { + int low = 0, high = ranges.Length / 2 - 1; + while (low <= high) + { + var mid = (low + high) >> 1; + if (point < ranges[2 * mid]) high = mid - 1; + else if (point > ranges[2 * mid + 1]) low = mid + 1; + else return true; + } + + return false; + } } } diff --git a/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs b/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs new file mode 100644 index 00000000..45cedbc5 --- /dev/null +++ b/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs @@ -0,0 +1,144 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +namespace GameCult.Networking +{ + internal static class CultNetUnicodeMessageIdTables + { + // (low, high) pairs, flattened and sorted. + public static readonly int[] LetterOrNumber = + { + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, + 0xD6, 0xD8, 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, + 0x376, 0x377, 0x37A, 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, + 0x3F5, 0x3F7, 0x481, 0x48A, 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, + 0x620, 0x64A, 0x660, 0x669, 0x66E, 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, + 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, + 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, + 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, + 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, + 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, + 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, 0xA5C, 0xA5E, 0xA5E, 0xA66, + 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, 0xAB3, 0xAB5, 0xAB9, + 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, 0xB10, 0xB13, + 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, 0xB6F, + 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, + 0xC12, 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, + 0xC7E, 0xC80, 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, + 0xCDD, 0xCDE, 0xCE0, 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, + 0xD3D, 0xD4E, 0xD4E, 0xD54, 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, + 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, + 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, + 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, + 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, + 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, + 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, 0x124A, 0x124D, 0x1250, 0x1256, + 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, 0x12B2, 0x12B5, 0x12B8, + 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, 0x1318, 0x135A, + 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, 0x1681, + 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, + 0x1819, 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, + 0x1946, 0x196D, 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, + 0x1A54, 0x1A80, 0x1A89, 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, + 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, + 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, + 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, + 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, + 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, + 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, 0x2107, 0x2107, 0x210A, 0x2113, + 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, 0x212D, 0x212F, + 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, 0x24EA, 0x24FF, + 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, 0x2D27, + 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, + 0x2E2F, 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, + 0x30A1, 0x30FA, 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, + 0x31FF, 0x3220, 0x3229, 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, + 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, + 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, + 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, + 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, + 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, + 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, + 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, 0xAAE0, 0xAAEA, 0xAAF2, + 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, 0xAB30, 0xAB5A, + 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0xF900, + 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, + 0xFD8F, 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, + 0xFF41, 0xFF5A, 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, + 0x1000B, 0x1000D, 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, + 0x10080, 0x100FA, 0x10107, 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, + 0x102D0, 0x102E1, 0x102FB, 0x10300, 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, + 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, + 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, + 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, 0x105B1, 0x105B3, 0x105B9, 0x105BB, + 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, 0x10780, 0x10785, + 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, 0x10837, + 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, + 0x10A60, 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, + 0x10B35, 0x10B40, 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, + 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, + 0x10D85, 0x10E60, 0x10E7E, 0x10E80, 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, + 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, + 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, + 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, 0x11144, 0x11147, 0x11147, 0x11150, + 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, 0x111DA, 0x111DC, 0x111DC, + 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, 0x11286, 0x11288, + 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, 0x112F9, + 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, + 0x1138E, 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, + 0x11434, 0x11447, 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, + 0x114C7, 0x114C7, 0x114D0, 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, + 0x11644, 0x11650, 0x11659, 0x11680, 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, + 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, + 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, + 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D0, 0x119E1, 0x119E1, 0x119E3, + 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, 0x11A50, 0x11A5C, 0x11A89, + 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, 0x11C0A, + 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, 0x11D09, + 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, + 0x11F12, 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, + 0x1246E, 0x12480, 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, + 0x14400, 0x14646, 0x16100, 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, + 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, + 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, + 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, + 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, + 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, 0x1B152, 0x1B155, 0x1B155, + 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, 0x1BC88, 0x1BC90, + 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, + 0x1D516, 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, + 0x1D550, 0x1D552, 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, + 0x1D716, 0x1D734, 0x1D736, 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, + 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, + 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, + 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, + 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, + 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, 0x1ECAB, 0x1ECAD, 0x1ECAF, + 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, 0x1EE21, + 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, 0x1EE39, + 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, + 0x1EE5D, 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, + 0x1EE72, 0x1EE74, 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, + 0x1EEA1, 0x1EEA3, 0x1EEA5, 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, + 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, + 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, 0x323AF, + }; + + public static readonly int[] DefaultIgnorable = + { + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, + 0x202A, 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, + 0xFFF8, 0x1BCA0, 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, + }; + } +} From 8598fd3d12f588541fe5a2e34179cdb84e35c14f Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:05:00 +0000 Subject: [PATCH 30/40] cultnet-rs: write_admitted messageId from the generated Unicode 16.0 tables (letters and numbers now 16.0, not 15.1) Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-rs/src/contracts.rs | 139 +---------------- packages/cultnet-rs/src/lib.rs | 1 + .../src/unicode_messageid_tables.rs | 147 ++++++++++++++++++ 3 files changed, 155 insertions(+), 132 deletions(-) create mode 100644 packages/cultnet-rs/src/unicode_messageid_tables.rs diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 201632a2..bf907806 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -934,137 +934,12 @@ fn require_non_empty(value: &str, field: &str) -> Result<()> { Ok(()) } -// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (\p{L}) or number (\p{N}) and no -// Default_Ignorable_Code_Point, compared raw (never trimmed). std has `char::is_numeric` (N) but no letter category -// (`is_alphabetic` is the Alphabetic property, which also takes combining marks), and no Default_Ignorable, so both tables -// are embedded: DerivedCoreProperties.txt ranges, Unicode 16.0 (unchanged since 15.1), and the general category L ranges of -// Unicode 15.1 (U+0378 is unassigned in every version, so it is refused either way). The Kotlin, Python and C# decoders embed -// the same Default_Ignorable ranges and take L/N from their platform; TypeScript uses its regex engine's tables. -// Invalid UTF-8 never reaches this check: the MessagePack str decode refuses it. -const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ - (0x00AD, 0x00AD), (0x034F, 0x034F), (0x061C, 0x061C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), - (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), - (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), -]; - -const LETTERS: &[(u32, u32)] = &[ - (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB5, 0xB5), (0xBA, 0xBA), (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), - (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), - (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), - (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), - (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), (0x6E5, 0x6E6), (0x6EE, 0x6EF), (0x6FA, 0x6FC), (0x6FF, 0x6FF), - (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), (0x7CA, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), - (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), - (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), (0x950, 0x950), (0x958, 0x961), (0x971, 0x980), - (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), - (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), (0x9F0, 0x9F1), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), - (0xA13, 0xA28), (0xA2A, 0xA30), (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), - (0xA72, 0xA74), (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), - (0xABD, 0xABD), (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), - (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB71, 0xB71), - (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), (0xB9E, 0xB9F), - (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xC05, 0xC0C), (0xC0E, 0xC10), (0xC12, 0xC28), - (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), (0xC80, 0xC80), (0xC85, 0xC8C), - (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), - (0xCF1, 0xCF2), (0xD04, 0xD0C), (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), - (0xD5F, 0xD61), (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), - (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), (0xE8C, 0xEA3), - (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), (0xEDC, 0xEDF), - (0xF00, 0xF00), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), (0x1000, 0x102A), (0x103F, 0x103F), - (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), (0x106E, 0x1070), (0x1075, 0x1081), - (0x108E, 0x108E), (0x10A0, 0x10C5), (0x10C7, 0x10C7), (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), - (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), - (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), - (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), - (0x1401, 0x166C), (0x166F, 0x167F), (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16F1, 0x16F8), (0x1700, 0x1711), - (0x171F, 0x1731), (0x1740, 0x1751), (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), - (0x17DC, 0x17DC), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), (0x18B0, 0x18F5), - (0x1900, 0x191E), (0x1950, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), (0x1A00, 0x1A16), - (0x1A20, 0x1A54), (0x1AA7, 0x1AA7), (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B83, 0x1BA0), (0x1BAE, 0x1BAF), - (0x1BBA, 0x1BE5), (0x1C00, 0x1C23), (0x1C4D, 0x1C4F), (0x1C5A, 0x1C7D), (0x1C80, 0x1C88), (0x1C90, 0x1CBA), - (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), - (0x1E00, 0x1F15), (0x1F18, 0x1F1D), (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), - (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), - (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), - (0x1FF6, 0x1FFC), (0x2071, 0x2071), (0x207F, 0x207F), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), - (0x210A, 0x2113), (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), - (0x212A, 0x212D), (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2183, 0x2184), - (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), - (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), - (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), - (0x3005, 0x3006), (0x3031, 0x3035), (0x303B, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), - (0x30FC, 0x30FF), (0x3105, 0x312F), (0x3131, 0x318E), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3400, 0x4DBF), - (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), (0xA500, 0xA60C), (0xA610, 0xA61F), (0xA62A, 0xA62B), (0xA640, 0xA66E), - (0xA67F, 0xA69D), (0xA6A0, 0xA6E5), (0xA717, 0xA71F), (0xA722, 0xA788), (0xA78B, 0xA7CA), (0xA7D0, 0xA7D1), - (0xA7D3, 0xA7D3), (0xA7D5, 0xA7D9), (0xA7F2, 0xA801), (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), - (0xA840, 0xA873), (0xA882, 0xA8B3), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA90A, 0xA925), - (0xA930, 0xA946), (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9CF), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9EF), - (0xA9FA, 0xA9FE), (0xAA00, 0xAA28), (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), - (0xAA7E, 0xAAAF), (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), - (0xAADB, 0xAADD), (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), - (0xAB20, 0xAB26), (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xAC00, 0xD7A3), - (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), - (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), - (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), - (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), (0xFFC2, 0xFFC7), - (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), (0x10028, 0x1003A), - (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), (0x10280, 0x1029C), - (0x102A0, 0x102D0), (0x10300, 0x1031F), (0x1032D, 0x10340), (0x10342, 0x10349), (0x10350, 0x10375), - (0x10380, 0x1039D), (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x10400, 0x1049D), (0x104B0, 0x104D3), - (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), (0x1057C, 0x1058A), - (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), (0x105B3, 0x105B9), - (0x105BB, 0x105BC), (0x10600, 0x10736), (0x10740, 0x10755), (0x10760, 0x10767), (0x10780, 0x10785), - (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), (0x10808, 0x10808), (0x1080A, 0x10835), - (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), (0x10860, 0x10876), (0x10880, 0x1089E), - (0x108E0, 0x108F2), (0x108F4, 0x108F5), (0x10900, 0x10915), (0x10920, 0x10939), (0x10980, 0x109B7), - (0x109BE, 0x109BF), (0x10A00, 0x10A00), (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), - (0x10A60, 0x10A7C), (0x10A80, 0x10A9C), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10B00, 0x10B35), - (0x10B40, 0x10B55), (0x10B60, 0x10B72), (0x10B80, 0x10B91), (0x10C00, 0x10C48), (0x10C80, 0x10CB2), - (0x10CC0, 0x10CF2), (0x10D00, 0x10D23), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10F00, 0x10F1C), - (0x10F27, 0x10F27), (0x10F30, 0x10F45), (0x10F70, 0x10F81), (0x10FB0, 0x10FC4), (0x10FE0, 0x10FF6), - (0x11003, 0x11037), (0x11071, 0x11072), (0x11075, 0x11075), (0x11083, 0x110AF), (0x110D0, 0x110E8), - (0x11103, 0x11126), (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), - (0x11183, 0x111B2), (0x111C1, 0x111C4), (0x111DA, 0x111DA), (0x111DC, 0x111DC), (0x11200, 0x11211), - (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), - (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x11305, 0x1130C), (0x1130F, 0x11310), - (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), (0x1133D, 0x1133D), - (0x11350, 0x11350), (0x1135D, 0x11361), (0x11400, 0x11434), (0x11447, 0x1144A), (0x1145F, 0x11461), - (0x11480, 0x114AF), (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x11580, 0x115AE), (0x115D8, 0x115DB), - (0x11600, 0x1162F), (0x11644, 0x11644), (0x11680, 0x116AA), (0x116B8, 0x116B8), (0x11700, 0x1171A), - (0x11740, 0x11746), (0x11800, 0x1182B), (0x118A0, 0x118DF), (0x118FF, 0x11906), (0x11909, 0x11909), - (0x1190C, 0x11913), (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), - (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), - (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), - (0x11AB0, 0x11AF8), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), (0x11C40, 0x11C40), (0x11C72, 0x11C8F), - (0x11D00, 0x11D06), (0x11D08, 0x11D09), (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D60, 0x11D65), - (0x11D67, 0x11D68), (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), - (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11FB0, 0x11FB0), (0x12000, 0x12399), (0x12480, 0x12543), - (0x12F90, 0x12FF0), (0x13000, 0x1342F), (0x13441, 0x13446), (0x14400, 0x14646), (0x16800, 0x16A38), - (0x16A40, 0x16A5E), (0x16A70, 0x16ABE), (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), - (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16E40, 0x16E7F), (0x16F00, 0x16F4A), (0x16F50, 0x16F50), - (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), (0x17000, 0x187F7), (0x18800, 0x18CD5), - (0x18D00, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), - (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), - (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), (0x1BC90, 0x1BC99), (0x1D400, 0x1D454), - (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), (0x1D4A9, 0x1D4AC), - (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), (0x1D507, 0x1D50A), - (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), (0x1D540, 0x1D544), - (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), (0x1D6C2, 0x1D6DA), - (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), (0x1D750, 0x1D76E), - (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), (0x1DF00, 0x1DF1E), - (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), (0x1E137, 0x1E13D), (0x1E14E, 0x1E14E), - (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), (0x1E4D0, 0x1E4EB), (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), - (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), - (0x1EE00, 0x1EE03), (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), - (0x1EE29, 0x1EE32), (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), - (0x1EE47, 0x1EE47), (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), - (0x1EE54, 0x1EE54), (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), - (0x1EE5F, 0x1EE5F), (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), - (0x1EE74, 0x1EE77), (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), - (0x1EEA1, 0x1EEA3), (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x20000, 0x2A6DF), (0x2A700, 0x2B739), - (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), - (0x30000, 0x3134A), (0x31350, 0x323AF), -]; +// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) and no +// Default_Ignorable_Code_Point, as of Unicode 16.0, compared raw (never trimmed). The tables are embedded +// (unicode_messageid_tables.rs, generated by scripts/gen-unicode-messageid-tables.py), not std's, so no platform's Unicode +// version changes a result; the other four runtimes embed the same generated tables. Invalid UTF-8 never reaches this check: +// the MessagePack str decode refuses it. +use crate::unicode_messageid_tables::{DEFAULT_IGNORABLE, LETTER_OR_NUMBER}; fn in_ranges(ranges: &[(u32, u32)], point: u32) -> bool { ranges.binary_search_by(|&(low, high)| { @@ -1086,7 +961,7 @@ fn require_admitted_message_id(value: &str) -> Result<()> { if in_ranges(DEFAULT_IGNORABLE, point) { return Err(anyhow!("CultNet field messageId must not hold a Default_Ignorable code point")); } - has_letter_or_number |= ch.is_numeric() || in_ranges(LETTERS, point); + has_letter_or_number |= in_ranges(LETTER_OR_NUMBER, point); } if !has_letter_or_number { return Err(anyhow!("CultNet field messageId must contain a letter or number")); diff --git a/packages/cultnet-rs/src/lib.rs b/packages/cultnet-rs/src/lib.rs index b35bd701..e008a2ba 100644 --- a/packages/cultnet-rs/src/lib.rs +++ b/packages/cultnet-rs/src/lib.rs @@ -3,6 +3,7 @@ mod contracts; mod idunn_deployment_brake; mod idunn_lifecycle_brake; mod media_fec; +mod unicode_messageid_tables; mod media_stream_contracts; mod media_stream_wire; mod provider_health_contracts; diff --git a/packages/cultnet-rs/src/unicode_messageid_tables.rs b/packages/cultnet-rs/src/unicode_messageid_tables.rs new file mode 100644 index 00000000..b5293f7a --- /dev/null +++ b/packages/cultnet-rs/src/unicode_messageid_tables.rs @@ -0,0 +1,147 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +// (low, high) pairs, sorted. +pub const LETTER_OR_NUMBER: &[(u32, u32)] = &[ + (0x30, 0x39), (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB2, 0xB3), (0xB5, 0xB5), (0xB9, 0xBA), (0xBC, 0xBE), + (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), + (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), + (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), + (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), (0x660, 0x669), (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), + (0x6E5, 0x6E6), (0x6EE, 0x6FC), (0x6FF, 0x6FF), (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), + (0x7C0, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), + (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), + (0x950, 0x950), (0x958, 0x961), (0x966, 0x96F), (0x971, 0x980), (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), + (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), + (0x9E6, 0x9F1), (0x9F4, 0x9F9), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), (0xA13, 0xA28), (0xA2A, 0xA30), + (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), (0xA66, 0xA6F), (0xA72, 0xA74), + (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), (0xABD, 0xABD), + (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAE6, 0xAEF), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), + (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB66, 0xB6F), + (0xB71, 0xB77), (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), + (0xB9E, 0xB9F), (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xBE6, 0xBF2), (0xC05, 0xC0C), + (0xC0E, 0xC10), (0xC12, 0xC28), (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), + (0xC66, 0xC6F), (0xC78, 0xC7E), (0xC80, 0xC80), (0xC85, 0xC8C), (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), + (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), (0xCE6, 0xCEF), (0xCF1, 0xCF2), (0xD04, 0xD0C), + (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), (0xD58, 0xD61), (0xD66, 0xD78), + (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), (0xDE6, 0xDEF), + (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE50, 0xE59), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), + (0xE8C, 0xEA3), (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), + (0xED0, 0xED9), (0xEDC, 0xEDF), (0xF00, 0xF00), (0xF20, 0xF33), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), + (0x1000, 0x102A), (0x103F, 0x1049), (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), + (0x106E, 0x1070), (0x1075, 0x1081), (0x108E, 0x108E), (0x1090, 0x1099), (0x10A0, 0x10C5), (0x10C7, 0x10C7), + (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), + (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), + (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), + (0x1369, 0x137C), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), (0x1401, 0x166C), (0x166F, 0x167F), + (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16EE, 0x16F8), (0x1700, 0x1711), (0x171F, 0x1731), (0x1740, 0x1751), + (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), (0x17DC, 0x17DC), (0x17E0, 0x17E9), + (0x17F0, 0x17F9), (0x1810, 0x1819), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), + (0x18B0, 0x18F5), (0x1900, 0x191E), (0x1946, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), + (0x19D0, 0x19DA), (0x1A00, 0x1A16), (0x1A20, 0x1A54), (0x1A80, 0x1A89), (0x1A90, 0x1A99), (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B50, 0x1B59), (0x1B83, 0x1BA0), (0x1BAE, 0x1BE5), (0x1C00, 0x1C23), + (0x1C40, 0x1C49), (0x1C4D, 0x1C7D), (0x1C80, 0x1C8A), (0x1C90, 0x1CBA), (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), (0x1E00, 0x1F15), (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), (0x1FF6, 0x1FFC), (0x2070, 0x2071), + (0x2074, 0x2079), (0x207F, 0x2089), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), (0x210A, 0x2113), + (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), (0x212A, 0x212D), + (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2150, 0x2189), (0x2460, 0x249B), + (0x24EA, 0x24FF), (0x2776, 0x2793), (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2CFD, 0x2CFD), + (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), (0x3005, 0x3007), (0x3021, 0x3029), (0x3031, 0x3035), + (0x3038, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), (0x30FC, 0x30FF), (0x3105, 0x312F), + (0x3131, 0x318E), (0x3192, 0x3195), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3220, 0x3229), (0x3248, 0x324F), + (0x3251, 0x325F), (0x3280, 0x3289), (0x32B1, 0x32BF), (0x3400, 0x4DBF), (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), (0xA610, 0xA62B), (0xA640, 0xA66E), (0xA67F, 0xA69D), (0xA6A0, 0xA6EF), (0xA717, 0xA71F), + (0xA722, 0xA788), (0xA78B, 0xA7CD), (0xA7D0, 0xA7D1), (0xA7D3, 0xA7D3), (0xA7D5, 0xA7DC), (0xA7F2, 0xA801), + (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), (0xA830, 0xA835), (0xA840, 0xA873), (0xA882, 0xA8B3), + (0xA8D0, 0xA8D9), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA900, 0xA925), (0xA930, 0xA946), + (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9D9), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9FE), (0xAA00, 0xAA28), + (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA50, 0xAA59), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xABF0, 0xABF9), (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), + (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF10, 0xFF19), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), + (0x10028, 0x1003A), (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), + (0x10107, 0x10133), (0x10140, 0x10178), (0x1018A, 0x1018B), (0x10280, 0x1029C), (0x102A0, 0x102D0), + (0x102E1, 0x102FB), (0x10300, 0x10323), (0x1032D, 0x1034A), (0x10350, 0x10375), (0x10380, 0x1039D), + (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x103D1, 0x103D5), (0x10400, 0x1049D), (0x104A0, 0x104A9), + (0x104B0, 0x104D3), (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), + (0x1057C, 0x1058A), (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), + (0x105B3, 0x105B9), (0x105BB, 0x105BC), (0x105C0, 0x105F3), (0x10600, 0x10736), (0x10740, 0x10755), + (0x10760, 0x10767), (0x10780, 0x10785), (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), + (0x10808, 0x10808), (0x1080A, 0x10835), (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), + (0x10858, 0x10876), (0x10879, 0x1089E), (0x108A7, 0x108AF), (0x108E0, 0x108F2), (0x108F4, 0x108F5), + (0x108FB, 0x1091B), (0x10920, 0x10939), (0x10980, 0x109B7), (0x109BC, 0x109CF), (0x109D2, 0x10A00), + (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), (0x10A40, 0x10A48), (0x10A60, 0x10A7E), + (0x10A80, 0x10A9F), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10AEB, 0x10AEF), (0x10B00, 0x10B35), + (0x10B40, 0x10B55), (0x10B58, 0x10B72), (0x10B78, 0x10B91), (0x10BA9, 0x10BAF), (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), (0x10CC0, 0x10CF2), (0x10CFA, 0x10D23), (0x10D30, 0x10D39), (0x10D40, 0x10D65), + (0x10D6F, 0x10D85), (0x10E60, 0x10E7E), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10EC2, 0x10EC4), + (0x10F00, 0x10F27), (0x10F30, 0x10F45), (0x10F51, 0x10F54), (0x10F70, 0x10F81), (0x10FB0, 0x10FCB), + (0x10FE0, 0x10FF6), (0x11003, 0x11037), (0x11052, 0x1106F), (0x11071, 0x11072), (0x11075, 0x11075), + (0x11083, 0x110AF), (0x110D0, 0x110E8), (0x110F0, 0x110F9), (0x11103, 0x11126), (0x11136, 0x1113F), + (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), (0x11183, 0x111B2), + (0x111C1, 0x111C4), (0x111D0, 0x111DA), (0x111DC, 0x111DC), (0x111E1, 0x111F4), (0x11200, 0x11211), + (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), + (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x112F0, 0x112F9), (0x11305, 0x1130C), + (0x1130F, 0x11310), (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), + (0x1133D, 0x1133D), (0x11350, 0x11350), (0x1135D, 0x11361), (0x11380, 0x11389), (0x1138B, 0x1138B), + (0x1138E, 0x1138E), (0x11390, 0x113B5), (0x113B7, 0x113B7), (0x113D1, 0x113D1), (0x113D3, 0x113D3), + (0x11400, 0x11434), (0x11447, 0x1144A), (0x11450, 0x11459), (0x1145F, 0x11461), (0x11480, 0x114AF), + (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x114D0, 0x114D9), (0x11580, 0x115AE), (0x115D8, 0x115DB), + (0x11600, 0x1162F), (0x11644, 0x11644), (0x11650, 0x11659), (0x11680, 0x116AA), (0x116B8, 0x116B8), + (0x116C0, 0x116C9), (0x116D0, 0x116E3), (0x11700, 0x1171A), (0x11730, 0x1173B), (0x11740, 0x11746), + (0x11800, 0x1182B), (0x118A0, 0x118F2), (0x118FF, 0x11906), (0x11909, 0x11909), (0x1190C, 0x11913), + (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), (0x11950, 0x11959), + (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), (0x11BC0, 0x11BE0), (0x11BF0, 0x11BF9), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), (0x11C50, 0x11C6C), (0x11C72, 0x11C8F), (0x11D00, 0x11D06), (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D50, 0x11D59), (0x11D60, 0x11D65), (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11DA0, 0x11DA9), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), + (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11F50, 0x11F59), (0x11FB0, 0x11FB0), (0x11FC0, 0x11FD4), + (0x12000, 0x12399), (0x12400, 0x1246E), (0x12480, 0x12543), (0x12F90, 0x12FF0), (0x13000, 0x1342F), + (0x13441, 0x13446), (0x13460, 0x143FA), (0x14400, 0x14646), (0x16100, 0x1611D), (0x16130, 0x16139), + (0x16800, 0x16A38), (0x16A40, 0x16A5E), (0x16A60, 0x16A69), (0x16A70, 0x16ABE), (0x16AC0, 0x16AC9), + (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), (0x16B50, 0x16B59), (0x16B5B, 0x16B61), + (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16D40, 0x16D6C), (0x16D70, 0x16D79), (0x16E40, 0x16E96), + (0x16F00, 0x16F4A), (0x16F50, 0x16F50), (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), (0x18800, 0x18CD5), (0x18CFF, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), + (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), (0x1CCF0, 0x1CCF9), (0x1D2C0, 0x1D2D3), (0x1D2E0, 0x1D2F3), (0x1D360, 0x1D378), + (0x1D400, 0x1D454), (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), + (0x1D7CE, 0x1D7FF), (0x1DF00, 0x1DF1E), (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), (0x1E140, 0x1E149), (0x1E14E, 0x1E14E), (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), + (0x1E2F0, 0x1E2F9), (0x1E4D0, 0x1E4EB), (0x1E4F0, 0x1E4F9), (0x1E5D0, 0x1E5ED), (0x1E5F0, 0x1E5FA), + (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), + (0x1E8C7, 0x1E8CF), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), (0x1E950, 0x1E959), (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), (0x1ECB1, 0x1ECB4), (0x1ED01, 0x1ED2D), (0x1ED2F, 0x1ED3D), (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x1F100, 0x1F10C), (0x1FBF0, 0x1FBF9), (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), (0x30000, 0x3134A), (0x31350, 0x323AF), +]; + +pub const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ + (0xAD, 0xAD), (0x34F, 0x34F), (0x61C, 0x61C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +]; From f378d4fdb095dc7f902c243724d8d0f6faf18456 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:11:15 +0000 Subject: [PATCH 31/40] tests first: duplicate messageId, ill-formed id, scoped UTF-8 strictness, every table range pinned (failing) Failing before the fix: C# 1 (duplicate messageId, second is U+200B), TypeScript 1 (a + lone surrogate), Kotlin 1 (a document payload string with a lone surrogate no longer decodes). Python and Rust already pass the new vectors. The range probes read the generated write-admitted-message-id-ranges.json and pass on the correct tables; they exist to fail when any range is deleted or narrowed. Co-Authored-By: Claude Sonnet 5.5 --- .../write-admitted-message-id-ranges.json | 1 + .../interop/write-receipt-vectors.json | 122 +++++++++++++++++- .../gamecult/cultmesh/WriteReceiptVectors.kt | 32 +++++ .../cultnet-py/tests/test_write_receipt.py | 26 ++++ packages/cultnet-rs/tests/write_receipt.rs | 37 ++++++ .../cultnet-ts/test/write-receipt.test.ts | 20 +++ scripts/gen-unicode-messageid-tables.py | 14 +- .../CultNetWriteAdmittedMessageIdTests.cs | 33 +++++ 8 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 contracts/cultnet/interop/write-admitted-message-id-ranges.json diff --git a/contracts/cultnet/interop/write-admitted-message-id-ranges.json b/contracts/cultnet/interop/write-admitted-message-id-ranges.json new file mode 100644 index 00000000..98ccdec0 --- /dev/null +++ b/contracts/cultnet/interop/write-admitted-message-id-ranges.json @@ -0,0 +1 @@ +{"generated":"generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit","unicodeVersion":"16.0.0","letterOrNumber":[[48,57],[65,90],[97,122],[170,170],[178,179],[181,181],[185,186],[188,190],[192,214],[216,246],[248,705],[710,721],[736,740],[748,748],[750,750],[880,884],[886,887],[890,893],[895,895],[902,902],[904,906],[908,908],[910,929],[931,1013],[1015,1153],[1162,1327],[1329,1366],[1369,1369],[1376,1416],[1488,1514],[1519,1522],[1568,1610],[1632,1641],[1646,1647],[1649,1747],[1749,1749],[1765,1766],[1774,1788],[1791,1791],[1808,1808],[1810,1839],[1869,1957],[1969,1969],[1984,2026],[2036,2037],[2042,2042],[2048,2069],[2074,2074],[2084,2084],[2088,2088],[2112,2136],[2144,2154],[2160,2183],[2185,2190],[2208,2249],[2308,2361],[2365,2365],[2384,2384],[2392,2401],[2406,2415],[2417,2432],[2437,2444],[2447,2448],[2451,2472],[2474,2480],[2482,2482],[2486,2489],[2493,2493],[2510,2510],[2524,2525],[2527,2529],[2534,2545],[2548,2553],[2556,2556],[2565,2570],[2575,2576],[2579,2600],[2602,2608],[2610,2611],[2613,2614],[2616,2617],[2649,2652],[2654,2654],[2662,2671],[2674,2676],[2693,2701],[2703,2705],[2707,2728],[2730,2736],[2738,2739],[2741,2745],[2749,2749],[2768,2768],[2784,2785],[2790,2799],[2809,2809],[2821,2828],[2831,2832],[2835,2856],[2858,2864],[2866,2867],[2869,2873],[2877,2877],[2908,2909],[2911,2913],[2918,2927],[2929,2935],[2947,2947],[2949,2954],[2958,2960],[2962,2965],[2969,2970],[2972,2972],[2974,2975],[2979,2980],[2984,2986],[2990,3001],[3024,3024],[3046,3058],[3077,3084],[3086,3088],[3090,3112],[3114,3129],[3133,3133],[3160,3162],[3165,3165],[3168,3169],[3174,3183],[3192,3198],[3200,3200],[3205,3212],[3214,3216],[3218,3240],[3242,3251],[3253,3257],[3261,3261],[3293,3294],[3296,3297],[3302,3311],[3313,3314],[3332,3340],[3342,3344],[3346,3386],[3389,3389],[3406,3406],[3412,3414],[3416,3425],[3430,3448],[3450,3455],[3461,3478],[3482,3505],[3507,3515],[3517,3517],[3520,3526],[3558,3567],[3585,3632],[3634,3635],[3648,3654],[3664,3673],[3713,3714],[3716,3716],[3718,3722],[3724,3747],[3749,3749],[3751,3760],[3762,3763],[3773,3773],[3776,3780],[3782,3782],[3792,3801],[3804,3807],[3840,3840],[3872,3891],[3904,3911],[3913,3948],[3976,3980],[4096,4138],[4159,4169],[4176,4181],[4186,4189],[4193,4193],[4197,4198],[4206,4208],[4213,4225],[4238,4238],[4240,4249],[4256,4293],[4295,4295],[4301,4301],[4304,4346],[4348,4680],[4682,4685],[4688,4694],[4696,4696],[4698,4701],[4704,4744],[4746,4749],[4752,4784],[4786,4789],[4792,4798],[4800,4800],[4802,4805],[4808,4822],[4824,4880],[4882,4885],[4888,4954],[4969,4988],[4992,5007],[5024,5109],[5112,5117],[5121,5740],[5743,5759],[5761,5786],[5792,5866],[5870,5880],[5888,5905],[5919,5937],[5952,5969],[5984,5996],[5998,6000],[6016,6067],[6103,6103],[6108,6108],[6112,6121],[6128,6137],[6160,6169],[6176,6264],[6272,6276],[6279,6312],[6314,6314],[6320,6389],[6400,6430],[6470,6509],[6512,6516],[6528,6571],[6576,6601],[6608,6618],[6656,6678],[6688,6740],[6784,6793],[6800,6809],[6823,6823],[6917,6963],[6981,6988],[6992,7001],[7043,7072],[7086,7141],[7168,7203],[7232,7241],[7245,7293],[7296,7306],[7312,7354],[7357,7359],[7401,7404],[7406,7411],[7413,7414],[7418,7418],[7424,7615],[7680,7957],[7960,7965],[7968,8005],[8008,8013],[8016,8023],[8025,8025],[8027,8027],[8029,8029],[8031,8061],[8064,8116],[8118,8124],[8126,8126],[8130,8132],[8134,8140],[8144,8147],[8150,8155],[8160,8172],[8178,8180],[8182,8188],[8304,8305],[8308,8313],[8319,8329],[8336,8348],[8450,8450],[8455,8455],[8458,8467],[8469,8469],[8473,8477],[8484,8484],[8486,8486],[8488,8488],[8490,8493],[8495,8505],[8508,8511],[8517,8521],[8526,8526],[8528,8585],[9312,9371],[9450,9471],[10102,10131],[11264,11492],[11499,11502],[11506,11507],[11517,11517],[11520,11557],[11559,11559],[11565,11565],[11568,11623],[11631,11631],[11648,11670],[11680,11686],[11688,11694],[11696,11702],[11704,11710],[11712,11718],[11720,11726],[11728,11734],[11736,11742],[11823,11823],[12293,12295],[12321,12329],[12337,12341],[12344,12348],[12353,12438],[12445,12447],[12449,12538],[12540,12543],[12549,12591],[12593,12686],[12690,12693],[12704,12735],[12784,12799],[12832,12841],[12872,12879],[12881,12895],[12928,12937],[12977,12991],[13312,19903],[19968,42124],[42192,42237],[42240,42508],[42512,42539],[42560,42606],[42623,42653],[42656,42735],[42775,42783],[42786,42888],[42891,42957],[42960,42961],[42963,42963],[42965,42972],[42994,43009],[43011,43013],[43015,43018],[43020,43042],[43056,43061],[43072,43123],[43138,43187],[43216,43225],[43250,43255],[43259,43259],[43261,43262],[43264,43301],[43312,43334],[43360,43388],[43396,43442],[43471,43481],[43488,43492],[43494,43518],[43520,43560],[43584,43586],[43588,43595],[43600,43609],[43616,43638],[43642,43642],[43646,43695],[43697,43697],[43701,43702],[43705,43709],[43712,43712],[43714,43714],[43739,43741],[43744,43754],[43762,43764],[43777,43782],[43785,43790],[43793,43798],[43808,43814],[43816,43822],[43824,43866],[43868,43881],[43888,44002],[44016,44025],[44032,55203],[55216,55238],[55243,55291],[63744,64109],[64112,64217],[64256,64262],[64275,64279],[64285,64285],[64287,64296],[64298,64310],[64312,64316],[64318,64318],[64320,64321],[64323,64324],[64326,64433],[64467,64829],[64848,64911],[64914,64967],[65008,65019],[65136,65140],[65142,65276],[65296,65305],[65313,65338],[65345,65370],[65382,65470],[65474,65479],[65482,65487],[65490,65495],[65498,65500],[65536,65547],[65549,65574],[65576,65594],[65596,65597],[65599,65613],[65616,65629],[65664,65786],[65799,65843],[65856,65912],[65930,65931],[66176,66204],[66208,66256],[66273,66299],[66304,66339],[66349,66378],[66384,66421],[66432,66461],[66464,66499],[66504,66511],[66513,66517],[66560,66717],[66720,66729],[66736,66771],[66776,66811],[66816,66855],[66864,66915],[66928,66938],[66940,66954],[66956,66962],[66964,66965],[66967,66977],[66979,66993],[66995,67001],[67003,67004],[67008,67059],[67072,67382],[67392,67413],[67424,67431],[67456,67461],[67463,67504],[67506,67514],[67584,67589],[67592,67592],[67594,67637],[67639,67640],[67644,67644],[67647,67669],[67672,67702],[67705,67742],[67751,67759],[67808,67826],[67828,67829],[67835,67867],[67872,67897],[67968,68023],[68028,68047],[68050,68096],[68112,68115],[68117,68119],[68121,68149],[68160,68168],[68192,68222],[68224,68255],[68288,68295],[68297,68324],[68331,68335],[68352,68405],[68416,68437],[68440,68466],[68472,68497],[68521,68527],[68608,68680],[68736,68786],[68800,68850],[68858,68899],[68912,68921],[68928,68965],[68975,68997],[69216,69246],[69248,69289],[69296,69297],[69314,69316],[69376,69415],[69424,69445],[69457,69460],[69488,69505],[69552,69579],[69600,69622],[69635,69687],[69714,69743],[69745,69746],[69749,69749],[69763,69807],[69840,69864],[69872,69881],[69891,69926],[69942,69951],[69956,69956],[69959,69959],[69968,70002],[70006,70006],[70019,70066],[70081,70084],[70096,70106],[70108,70108],[70113,70132],[70144,70161],[70163,70187],[70207,70208],[70272,70278],[70280,70280],[70282,70285],[70287,70301],[70303,70312],[70320,70366],[70384,70393],[70405,70412],[70415,70416],[70419,70440],[70442,70448],[70450,70451],[70453,70457],[70461,70461],[70480,70480],[70493,70497],[70528,70537],[70539,70539],[70542,70542],[70544,70581],[70583,70583],[70609,70609],[70611,70611],[70656,70708],[70727,70730],[70736,70745],[70751,70753],[70784,70831],[70852,70853],[70855,70855],[70864,70873],[71040,71086],[71128,71131],[71168,71215],[71236,71236],[71248,71257],[71296,71338],[71352,71352],[71360,71369],[71376,71395],[71424,71450],[71472,71483],[71488,71494],[71680,71723],[71840,71922],[71935,71942],[71945,71945],[71948,71955],[71957,71958],[71960,71983],[71999,71999],[72001,72001],[72016,72025],[72096,72103],[72106,72144],[72161,72161],[72163,72163],[72192,72192],[72203,72242],[72250,72250],[72272,72272],[72284,72329],[72349,72349],[72368,72440],[72640,72672],[72688,72697],[72704,72712],[72714,72750],[72768,72768],[72784,72812],[72818,72847],[72960,72966],[72968,72969],[72971,73008],[73030,73030],[73040,73049],[73056,73061],[73063,73064],[73066,73097],[73112,73112],[73120,73129],[73440,73458],[73474,73474],[73476,73488],[73490,73523],[73552,73561],[73648,73648],[73664,73684],[73728,74649],[74752,74862],[74880,75075],[77712,77808],[77824,78895],[78913,78918],[78944,82938],[82944,83526],[90368,90397],[90416,90425],[92160,92728],[92736,92766],[92768,92777],[92784,92862],[92864,92873],[92880,92909],[92928,92975],[92992,92995],[93008,93017],[93019,93025],[93027,93047],[93053,93071],[93504,93548],[93552,93561],[93760,93846],[93952,94026],[94032,94032],[94099,94111],[94176,94177],[94179,94179],[94208,100343],[100352,101589],[101631,101640],[110576,110579],[110581,110587],[110589,110590],[110592,110882],[110898,110898],[110928,110930],[110933,110933],[110948,110951],[110960,111355],[113664,113770],[113776,113788],[113792,113800],[113808,113817],[118000,118009],[119488,119507],[119520,119539],[119648,119672],[119808,119892],[119894,119964],[119966,119967],[119970,119970],[119973,119974],[119977,119980],[119982,119993],[119995,119995],[119997,120003],[120005,120069],[120071,120074],[120077,120084],[120086,120092],[120094,120121],[120123,120126],[120128,120132],[120134,120134],[120138,120144],[120146,120485],[120488,120512],[120514,120538],[120540,120570],[120572,120596],[120598,120628],[120630,120654],[120656,120686],[120688,120712],[120714,120744],[120746,120770],[120772,120779],[120782,120831],[122624,122654],[122661,122666],[122928,122989],[123136,123180],[123191,123197],[123200,123209],[123214,123214],[123536,123565],[123584,123627],[123632,123641],[124112,124139],[124144,124153],[124368,124397],[124400,124410],[124896,124902],[124904,124907],[124909,124910],[124912,124926],[124928,125124],[125127,125135],[125184,125251],[125259,125259],[125264,125273],[126065,126123],[126125,126127],[126129,126132],[126209,126253],[126255,126269],[126464,126467],[126469,126495],[126497,126498],[126500,126500],[126503,126503],[126505,126514],[126516,126519],[126521,126521],[126523,126523],[126530,126530],[126535,126535],[126537,126537],[126539,126539],[126541,126543],[126545,126546],[126548,126548],[126551,126551],[126553,126553],[126555,126555],[126557,126557],[126559,126559],[126561,126562],[126564,126564],[126567,126570],[126572,126578],[126580,126583],[126585,126588],[126590,126590],[126592,126601],[126603,126619],[126625,126627],[126629,126633],[126635,126651],[127232,127244],[130032,130041],[131072,173791],[173824,177977],[177984,178205],[178208,183969],[183984,191456],[191472,192093],[194560,195101],[196608,201546],[201552,205743]],"defaultIgnorable":[[173,173],[847,847],[1564,1564],[4447,4448],[6068,6069],[6155,6159],[8203,8207],[8234,8238],[8288,8303],[12644,12644],[65024,65039],[65279,65279],[65440,65440],[65520,65528],[113824,113827],[119155,119162],[917504,921599]]} diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index c7f47738..f99265ed 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -150,7 +150,7 @@ ] }, "admittedMessageId": { - "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Letters, numbers and Default_Ignorable are those of Unicode 16.0 in every runtime, from embedded tables (no platform category lookup), so a runtime's own Unicode version cannot change a result. Scope: write_admitted only; error.v1 is untouched.", + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. Every messageId key of a frame must be valid. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string or this file cannot hold (a lone surrogate is not valid UTF-8; the bytes are ED A0 80) or a frame that repeats a key. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses earlier; byte-level strictness is a property of the reader and is not required of a decoder. Scope: write_admitted only; error.v1 is untouched.", "cases": [ { "name": "refuse_braille_blank_2800", @@ -222,7 +222,7 @@ "D800" ], "expect": "refuse", - "why": "not a Unicode scalar value; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str, so the decoder must refuse the frame. Rust, C#, Kotlin and Python build the frame from wireHex; a runtime whose strings can hold a lone surrogate (TypeScript) may also build it from the code point", + "why": "a lone surrogate is not a Unicode scalar value and has no letter or number; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str. A reader that rejects invalid UTF-8 refuses the frame; one that substitutes or keeps the surrogate refuses the id", "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" }, { @@ -408,6 +408,122 @@ "expect": "refuse", "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0" }, + { + "name": "refuse_duplicate_message_id_second_is_default_ignorable", + "codePoints": ["0061"], + "expect": "refuse", + "why": "the frame carries messageId twice, \"a\" and then U+200B; every messageId key must be valid, and a reader that keeps the last one sees U+200B", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B" + }, + { + "name": "refuse_letter_then_lone_surrogate_D800", + "codePoints": ["0061"], + "expect": "refuse", + "why": "\"a\" followed by a lone surrogate: the id as decoded is not well-formed Unicode. A reader that rejects invalid UTF-8 refuses the frame earlier; one that substitutes or keeps the surrogate must still refuse the id. Byte-level strictness is a property of the reader, not required of the decoder", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A461EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "accept_astral_mathematical_bold_capital_a_1D400", + "codePoints": ["1D400"], + "expect": "accept", + "why": "an astral letter (Lu)" + }, + { + "name": "accept_astral_cjk_ext_b_20000", + "codePoints": ["20000"], + "expect": "accept", + "why": "an astral letter (Lo)" + }, + { + "name": "refuse_letter_then_default_ignorable_061C", + "codePoints": ["0061", "061C"], + "expect": "refuse", + "why": "U+061C is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B4", + "codePoints": ["0061", "17B4"], + "expect": "refuse", + "why": "U+17B4 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B5", + "codePoints": ["0061", "17B5"], + "expect": "refuse", + "why": "U+17B5 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202A", + "codePoints": ["0061", "202A"], + "expect": "refuse", + "why": "U+202A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202E", + "codePoints": ["0061", "202E"], + "expect": "refuse", + "why": "U+202E is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE00", + "codePoints": ["0061", "FE00"], + "expect": "refuse", + "why": "U+FE00 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE0F", + "codePoints": ["0061", "FE0F"], + "expect": "refuse", + "why": "U+FE0F is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF0", + "codePoints": ["0061", "FFF0"], + "expect": "refuse", + "why": "U+FFF0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF8", + "codePoints": ["0061", "FFF8"], + "expect": "refuse", + "why": "U+FFF8 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA0", + "codePoints": ["0061", "1BCA0"], + "expect": "refuse", + "why": "U+1BCA0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA3", + "codePoints": ["0061", "1BCA3"], + "expect": "refuse", + "why": "U+1BCA3 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D173", + "codePoints": ["0061", "1D173"], + "expect": "refuse", + "why": "U+1D173 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D17A", + "codePoints": ["0061", "1D17A"], + "expect": "refuse", + "why": "U+1D17A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0000", + "codePoints": ["0061", "E0000"], + "expect": "refuse", + "why": "U+E0000 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0FFF", + "codePoints": ["0061", "E0FFF"], + "expect": "refuse", + "why": "U+E0FFF is Default_Ignorable (an end of one of its ranges)" + }, { "name": "accept_abc", "codePoints": [ @@ -1524,7 +1640,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 40 + "admittedMessageIdCases": 59 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index 8955c52d..35e0136f 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -68,6 +68,38 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every + // range, so deleting or narrowing any range fails here. + check("unicodeTableRanges") { + @Suppress("UNCHECKED_CAST") + val ranges = JsonReader(File(File(path).parentFile, "write-admitted-message-id-ranges.json").readText(Charsets.UTF_8)).readDocument() as Map + fun pairs(key: String) = list(ranges[key]).map { list(it).map { n -> (n as Long).toInt() } } + val letters = pairs("letterOrNumber") + val ignorable = pairs("defaultIgnorable") + val problems = mutableListOf() + if (letters.size <= 700 || ignorable.size != 17) problems += "ranges file holds ${letters.size} and ${ignorable.size} ranges" + fun accepted(id: String): Boolean = try { + decodeReceiptFrame(parseCultNetMessage(MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray())) + true + } catch (_: IOException) { false } + for ((low, high) in letters) for ((point, want) in listOf(low to true, high to true, low - 1 to false, high + 1 to false)) { + if (accepted(String(Character.toChars(point))) != want) problems += "U+${point.toString(16)} alone should be ${if (want) "accepted" else "refused"}" + } + for ((low, high) in ignorable) for ((point, want) in listOf(low to false, high to false, low - 1 to true, high + 1 to true)) { + if (accepted("a" + String(Character.toChars(point))) != want) problems += "a + U+${point.toString(16)} should be ${if (want) "accepted" else "refused"}" + } + problems + } + + // A document payload string with a lone surrogate (ED A0 80) is not a write_admitted id: the shared MessagePack reader keeps + // reading it. Strictness is scoped to the write_admitted frame, never to the reader. + check("documentPayloadStringWithLoneSurrogateStillDecodes") { + val text = MessagePackReader(unhex("A3EDA080")).readString() + if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string") + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 0c42a6af..fce0d850 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -94,6 +94,32 @@ class WriteReceiptVectorTests(unittest.TestCase): failures.append(f"{case['name']}: expected {case['expect']}") self.assertEqual(failures, []) + def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None: + # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and + # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and + # last point is refused and beside the points either side accepted. Deleting or narrowing any embedded range fails here. + ranges = json.loads((_INTEROP / "write-admitted-message-id-ranges.json").read_text(encoding="utf-8")) + self.assertGreater(len(ranges["letterOrNumber"]), 700) + self.assertEqual(len(ranges["defaultIgnorable"]), 17) + + def accepted(text: str) -> bool: + try: + parse_message(_pack(_admitted(messageId=text))) + return True + except ValueError: + return False + + failures = [] + for low, high in ranges["letterOrNumber"]: + for point, want in ((low, True), (high, True), (low - 1, False), (high + 1, False)): + if accepted(chr(point)) != want: + failures.append(f"U+{point:04X} alone should be {'accepted' if want else 'refused'}") + for low, high in ranges["defaultIgnorable"]: + for point, want in ((low, False), (high, False), (low - 1, True), (high + 1, True)): + if accepted("a" + chr(point)) != want: + failures.append(f"a + U+{point:04X} should be {'accepted' if want else 'refused'}") + self.assertEqual(failures, []) + def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: payload = None if case["payloadHex"] is None else bytes.fromhex(case["payloadHex"]) diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 8dccf996..10d7dcb8 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -201,3 +201,40 @@ fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() { } assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } + +/// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +#[test] +fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-admitted-message-id-ranges.json"); + let ranges: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let letters = ranges["letterOrNumber"].as_array().unwrap(); + let ignorable = ranges["defaultIgnorable"].as_array().unwrap(); + assert!(letters.len() > 700); + assert_eq!(ignorable.len(), 17); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let accepted = |id: &str| { + let header = vec![0xA0 | id.len() as u8]; + let frame = [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat(); + decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0).is_ok() + }; + let point = |value: i64| char::from_u32(value as u32).unwrap().to_string(); + let mut failures = Vec::new(); + for range in letters { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, true), (high, true), (low - 1, false), (high + 1, false)] { + if accepted(&point(at)) != want { + failures.push(format!("U+{at:X} alone should be {}", if want { "accepted" } else { "refused" })); + } + } + } + for range in ignorable { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, false), (high, false), (low - 1, true), (high + 1, true)] { + if accepted(&format!("a{}", point(at))) != want { + failures.push(format!("a + U+{at:X} should be {}", if want { "accepted" } else { "refused" })); + } + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index c447367a..c654de1a 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -303,3 +303,23 @@ test("write_admitted messageId: every admittedMessageId vector is refused or acc } assert.deepEqual(failures, []); }); + +// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned at both ends", () => { + const ranges = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"), "utf8")); + assert.ok(ranges.letterOrNumber.length > 700); + assert.equal(ranges.defaultIgnorable.length, 17); + const accepted = (id: string) => { try { parseCultNetMessage(admitted(id)); return true; } catch { return false; } }; + const failures: string[] = []; + for (const [low, high] of ranges.letterOrNumber as [number, number][]) { + for (const [point, want] of [[low, true], [high, true], [low - 1, false], [high + 1, false]] as [number, boolean][]) { + if (accepted(String.fromCodePoint(point)) !== want) failures.push(`U+${point.toString(16)} alone should be ${want ? "accepted" : "refused"}`); + } + } + for (const [low, high] of ranges.defaultIgnorable as [number, number][]) { + for (const [point, want] of [[low, false], [high, false], [low - 1, true], [high + 1, true]] as [number, boolean][]) { + if (accepted("a" + String.fromCodePoint(point)) !== want) failures.push(`a + U+${point.toString(16)} should be ${want ? "accepted" : "refused"}`); + } + } + assert.deepEqual(failures, []); +}); diff --git a/scripts/gen-unicode-messageid-tables.py b/scripts/gen-unicode-messageid-tables.py index 96d1a205..4c119930 100644 --- a/scripts/gen-unicode-messageid-tables.py +++ b/scripts/gen-unicode-messageid-tables.py @@ -13,6 +13,7 @@ https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --chec any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only. """ import hashlib +import json import re import sys from pathlib import Path @@ -28,6 +29,9 @@ TARGETS = { "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs", "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs", } +# The independent copy the runtimes' tests probe (first, last, and the code point either side of every range), so that deleting +# or narrowing any range of any embedded table fails a test. +RANGES_JSON = ROOT / "contracts/cultnet/interop/write-admitted-message-id-ranges.json" def parse(path, wanted): @@ -95,8 +99,12 @@ def main(): digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16] header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit" stale = [] - for target, path in TARGETS.items(): - text = emit(target, letters, ignorable, header) + texts = {path: emit(target, letters, ignorable, header) for target, path in TARGETS.items()} + texts[RANGES_JSON] = json.dumps( + {"generated": header, "unicodeVersion": VERSION, "letterOrNumber": [list(r) for r in letters], "defaultIgnorable": [list(r) for r in ignorable]}, + separators=(",", ":"), + ) + "\n" + for path, text in texts.items(): if check: if not path.exists() or path.read_text(encoding="utf-8") != text: stale.append(str(path.relative_to(ROOT))) @@ -104,7 +112,7 @@ def main(): path.write_text(text, encoding="utf-8") if stale: sys.exit("stale or hand-edited: " + ", ".join(stale)) - print(f"{'checked' if check else 'wrote'} {len(TARGETS)} tables: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") + print(f"{'checked' if check else 'wrote'} {len(texts)} files: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") if __name__ == "__main__": diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index f5cac528..a4f1475c 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -54,5 +54,38 @@ namespace GameCult.Networking.Tests Assert.That(decoded!.MessageId, Is.EqualTo(id)); } } + + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. + [Test] + public void EveryRangeOfTheUnicodeTablesIsPinnedAtBothEnds() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var ranges = JsonDocument.Parse(File.ReadAllText(Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"))).RootElement; + var letters = ranges.GetProperty("letterOrNumber").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + var ignorable = ranges.GetProperty("defaultIgnorable").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + Assert.That(letters.Length, Is.GreaterThan(700)); + Assert.That(ignorable.Length, Is.EqualTo(17)); + static bool Accepted(string id) + { + try + { + CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize( + new CultNetWriteAdmittedMessage { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" })); + return true; + } + catch (Exception) { return false; } + } + + var failures = new List(); + foreach (var (low, high) in letters) + foreach (var (point, want) in new[] { (low, true), (high, true), (low - 1, false), (high + 1, false) }) + if (Accepted(char.ConvertFromUtf32(point)) != want) failures.Add($"U+{point:X} alone should be {(want ? "accepted" : "refused")}"); + foreach (var (low, high) in ignorable) + foreach (var (point, want) in new[] { (low, false), (high, false), (low - 1, true), (high + 1, true) }) + if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}"); + Assert.That(failures, Is.Empty); + } } } From 2f372eb2f1658696f7115474bd4d7777246009fc Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:11:53 +0000 Subject: [PATCH 32/40] GameCult.Networking: check every messageId key of a write_admitted frame, not only the first Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetSchemaMessageSerialization.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index 3069384a..cb33b707 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -111,7 +111,7 @@ namespace GameCult.Networking { private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); - /// Throws unless the frame's top-level messageId is a valid write_admitted id; other keys are left to the serializer. + /// Throws unless every top-level messageId key of the frame is a valid write_admitted id; other keys are left to the serializer. public static void Require(byte[] payload) { var reader = new MessagePackReader(payload); @@ -125,7 +125,7 @@ namespace GameCult.Networking try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); - return; + continue; // a repeated messageId key is checked too: the serializer keeps the last, so every one must be valid } if (key == null) reader.Skip(); From 99273f468c5441ba67fa527c5834e3c2264ff7aa Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:12:07 +0000 Subject: [PATCH 33/40] cultnet-ts: refuse a write_admitted messageId that is not well-formed Unicode (lone surrogate) Co-Authored-By: Claude Sonnet 5.5 --- packages/cultnet-ts/src/contracts.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 09d5d823..8dd37ced 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -1478,7 +1478,8 @@ function isValidAdmittedMessageId(value: unknown): boolean { let hasLetterOrNumber = false; for (const ch of value) { const point = ch.codePointAt(0) as number; - if (inRanges(DEFAULT_IGNORABLE, point)) { + // The decoded string must be well-formed Unicode: @msgpack/msgpack turns invalid UTF-8 (ED A0 80) into a lone surrogate. + if ((point >= 0xd800 && point <= 0xdfff) || inRanges(DEFAULT_IGNORABLE, point)) { return false; } hasLetterOrNumber = hasLetterOrNumber || inRanges(LETTER_OR_NUMBER, point); From 6c130581a1193a810584a18b279d6f1fdc51c3f7 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:13:01 +0000 Subject: [PATCH 34/40] cultmesh-kotlin: UTF-8 strictness scoped to the write_admitted frame, not the shared MessagePack reader Co-Authored-By: Claude Sonnet 5.5 --- .../src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 2337448d..72c1d41d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -2365,6 +2365,9 @@ fun parseCultNetMessage(payload: ByteArray): CultNetMessage { if (schemaVersion !is String || schemaVersion.isBlank()) { throw IOException("CultNet schema-v0 messages must declare schemaVersion") } + // A write_admitted id is echoed byte for byte, so that frame alone is read again with invalid UTF-8 refused; document payload + // strings elsewhere keep decoding as before. + if (schemaVersion == WRITE_ADMITTED_VERSION) MessagePackReader(payload, strictUtf8 = true).readAny() val body = linkedMapOf() for ((key, value) in decoded) { if (key !is String) throw IOException("CultNet schema-v0 message keys must be strings") @@ -6152,7 +6155,8 @@ private fun requireSchemaResponse(message: CultNetMessage?, expectedSchemaVersio return response } -class MessagePackReader(payload: ByteArray) { +/** [strictUtf8] makes a str whose bytes are not UTF-8 throw instead of becoming U+FFFD; only the write_admitted frame asks for it. */ +class MessagePackReader(payload: ByteArray, private val strictUtf8: Boolean = false) { private val input = DataInputStream(ByteArrayInputStream(payload)) private var pushed = -1 @@ -6307,9 +6311,10 @@ class MessagePackReader(payload: ByteArray) { code == 0xdb -> input.readInt() else -> throw IOException("expected string") } - // Strict: bytes that are not UTF-8 (a lone surrogate's ED A0 80) refuse the frame instead of becoming U+FFFD. + val bytes = readPayload(length) + if (!strictUtf8) return String(bytes, StandardCharsets.UTF_8) return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT) - .decode(ByteBuffer.wrap(readPayload(length))).toString() + .decode(ByteBuffer.wrap(bytes)).toString() } private fun readBinaryAfterCode(code: Int): ByteArray { From a97861945c914bb2d08cc95174aa65d2366f196c Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:13:09 +0000 Subject: [PATCH 35/40] write-receipt.md: the messageId rule is on the decoded string; byte-level UTF-8 strictness is the reader's, not required Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 10f42a6a..4f6cb7d1 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. Every `messageId` key of a frame must be valid. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. @@ -67,7 +67,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; the lone surrogate U+D800 is carried as a whole frame in `wireHex`, because it is not valid UTF-8 and only some runtimes' strings can hold it), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; a lone surrogate, and a frame that repeats the key, are carried as whole frames in `wireHex`, because a lone surrogate is not valid UTF-8 and only some runtimes' strings can hold it; `write-admitted-message-id-ranges.json` lists every range of the embedded tables, and each runtime probes the first and last point of each range and the points either side), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance status From b55521d50dc91c00587d48eab8d5cef323d1d2f9 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 15:50:10 +0000 Subject: [PATCH 36/40] cultmesh-ts test: observe queued provider registers from creation so a client timeout is not an unhandled rejection Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-ts/test/provider-rudp-transport.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index aa4db341..577acc8a 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -287,18 +287,21 @@ test("a replaced RUDP generation drops operations queued behind an active handle runtimeId: identity.serviceInstanceId, connectionId, sessionToken: "signed-provider-token", - operationTimeoutMs: 100, + operationTimeoutMs: 1, // expires long before the test ends: the settle-by-timeout path is always exercised socketFactory: () => socket, }).connect(identity, new AbortController().signal); try { + // The handler is held open and the closed generation never answers, so these two settle only by the + // client's operation timeout or by connection.close(). Observe them from the start: whichever fires + // first is a legitimate end of the call, and this test asserts on server-side handler calls, not on it. const first = connection.register( { identity, requestedLeaseDurationMs: 10_000 }, new AbortController().signal, - ); + ).then(() => undefined, () => undefined); const second = connection.register( { identity, requestedLeaseDurationMs: 10_000 }, new AbortController().signal, - ); + ).then(() => undefined, () => undefined); await waitFor(() => handlerCalls === 1, "first queued provider operation"); const replacement = new CultNetRudpSession({ connectionId }); @@ -318,7 +321,7 @@ test("a replaced RUDP generation drops operations queued behind an active handle assert.equal(handlerCalls, 1, "transport-queued work from the closed generation must be dropped"); connection.close(); - await Promise.allSettled([first, second]); + await Promise.all([first, second]); } finally { connection.close(); server.close(); From cc5a9cec9b446996859212625a0413b6cdd88b10 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:04:58 +0000 Subject: [PATCH 37/40] tests first: repeated messageId key, each runtime's declared policy (C# failing) Vectors admittedMessageIdDuplicates carry [invalid,valid], [valid,invalid], [valid,valid]. Decoders that see the duplicate (C#, Rust) must refuse all three; map-collapsing decoders (Python, TS, Kotlin) judge the last key. Python, TS, Kotlin and Rust already behave as declared; C# fails: it accepted [valid,valid]. Co-Authored-By: Claude Sonnet 5.5 --- .../interop/write-receipt-vectors.json | 28 ++++++++++++++++++- .../gamecult/cultmesh/WriteReceiptVectors.kt | 14 ++++++++++ .../cultnet-py/tests/test_write_receipt.py | 14 +++++++++- packages/cultnet-rs/tests/write_receipt.rs | 14 ++++++++++ .../cultnet-ts/test/write-receipt.test.ts | 12 ++++++++ .../CultNetWriteAdmittedMessageIdTests.cs | 14 ++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index f99265ed..e95c3180 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -606,6 +606,31 @@ } ] }, + "admittedMessageIdDuplicates": { + "semantics": "A frame with more than one messageId key is malformed. A decoder that can see the duplicate (C#, Rust: raw MessagePack reader or ordered pairs) refuses it, whatever the values (`seesDuplicate`). A decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses exactly when that value is invalid (`lastKeyWins`); where it accepts it echoes the last key's id (`lastKeyWinsEcho`). Neither behaviour ever echoes an invalid id. Each runtime declares which policy it has and its runner asserts that policy's column, so a change of a decoder's key policy fails a named test. The first key is ZWSP (U+200B, Default_Ignorable) in `invalid_then_valid`; the second key is the last entry of the map. Frames are MessagePack hex.", + "cases": [ + { + "name": "invalid_then_valid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3E2808BA96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A161", + "seesDuplicate": "refuse", + "lastKeyWins": "accept", + "lastKeyWinsEcho": "a" + }, + { + "name": "valid_then_invalid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A161A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B", + "seesDuplicate": "refuse", + "lastKeyWins": "refuse" + }, + { + "name": "valid_then_valid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A161A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A162", + "seesDuplicate": "refuse", + "lastKeyWins": "accept", + "lastKeyWinsEcho": "b" + } + ] + }, "schemaInvalid": [ { "name": "write_admitted_with_an_error_key", @@ -1640,7 +1665,8 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 59 + "admittedMessageIdCases": 59, + "admittedMessageIdDuplicateCases": 3 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index 35e0136f..b7215a12 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -100,6 +100,20 @@ fun replayWriteReceiptVectors(path: String): Boolean { if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string") } + // admittedMessageIdDuplicates: Kotlin decodes the frame to a map, so it cannot see a repeated key: it judges the last one. + for (case in list(obj(vectors["admittedMessageIdDuplicates"])["cases"])) { + val c = obj(case) + check("admittedMessageIdDuplicates/${c["name"]}") { + val decoded = try { decodeReceiptFrame(parseCultNetMessage(unhex(c["wireHex"] as String))) } catch (_: IOException) { null } + when { + c["lastKeyWins"] == "refuse" -> if (decoded == null) emptyList() else listOf("accepted a frame whose last messageId key is invalid") + decoded == null -> listOf("refused a frame whose last messageId key is valid") + decoded.toWireMap()["messageId"] != c["lastKeyWinsEcho"] -> listOf("echoed ${decoded.toWireMap()["messageId"]}, wanted ${c["lastKeyWinsEcho"]}") + else -> emptyList() + } + } + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index fce0d850..0ff1d017 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -57,7 +57,7 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId", "admittedMessageIdDuplicates"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) @@ -94,6 +94,18 @@ class WriteReceiptVectorTests(unittest.TestCase): failures.append(f"{case['name']}: expected {case['expect']}") self.assertEqual(failures, []) + def test_a_repeated_message_id_key_follows_the_last_key_wins_policy(self) -> None: + # Python decodes the frame to a dict, so it cannot see a duplicate: it judges the last key (admittedMessageIdDuplicates). + section = _VECTORS["admittedMessageIdDuplicates"] + self.assertEqual(_VECTORS["counts"]["admittedMessageIdDuplicateCases"], len(section["cases"])) + for case in section["cases"]: + raw = bytes.fromhex(case["wireHex"]) + if case["lastKeyWins"] == "refuse": + with self.assertRaises(ValueError, msg=case["name"]): + parse_message(raw) + else: + self.assertEqual(parse_message(raw).body["messageId"], case["lastKeyWinsEcho"], case["name"]) + def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None: # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 10d7dcb8..ee05bd72 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -238,3 +238,17 @@ fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { } assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } + +/// Vectors `admittedMessageIdDuplicates`: Rust decodes a map it can see into, so a repeated messageId key is refused whatever the values. +#[test] +fn a_repeated_message_id_key_is_refused_whatever_the_values() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + let vectors: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = vectors["admittedMessageIdDuplicates"]["cases"].as_array().unwrap(); + assert_eq!(vectors["counts"]["admittedMessageIdDuplicateCases"].as_u64().unwrap() as usize, cases.len()); + for case in cases { + assert_eq!(case["seesDuplicate"], "refuse"); + let decoded = decode_cultnet_message_from_slice(&unhex(case["wireHex"].as_str().unwrap()), CultNetWireContract::CultNetSchemaV0); + assert!(decoded.is_err(), "{}: a repeated messageId key must be refused", case["name"]); + } +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index c654de1a..1483fe31 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -323,3 +323,15 @@ test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned } assert.deepEqual(failures, []); }); + +// TypeScript decodes the frame to an object, so it cannot see a duplicate key: it judges the last one (admittedMessageIdDuplicates). +test("write_admitted messageId: a repeated key follows the last-key-wins policy", () => { + const vectors = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); + const section = vectors.admittedMessageIdDuplicates; + assert.equal(vectors.counts.admittedMessageIdDuplicateCases, section.cases.length); + for (const vector of section.cases as { name: string; wireHex: string; lastKeyWins: "accept" | "refuse"; lastKeyWinsEcho?: string }[]) { + const frame = decode(Uint8Array.from(Buffer.from(vector.wireHex, "hex"))); + if (vector.lastKeyWins === "refuse") assert.throws(() => parseCultNetMessage(frame), /./s, vector.name); + else assert.equal((parseCultNetMessage(frame) as CultNetWriteAdmittedMessage).messageId, vector.lastKeyWinsEcho, vector.name); + } +}); diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index a4f1475c..6c65734c 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -87,5 +87,19 @@ namespace GameCult.Networking.Tests if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}"); Assert.That(failures, Is.Empty); } + + // admittedMessageIdDuplicates: C# reads the raw map, so it sees a repeated messageId key and refuses it whatever the values. + [Test] + public void ARepeatedMessageIdKeyIsRefusedWhateverTheValues() + { + var cases = Vectors().GetProperty("admittedMessageIdDuplicates").GetProperty("cases").EnumerateArray().ToArray(); + Assert.That(cases.Length, Is.EqualTo(Vectors().GetProperty("counts").GetProperty("admittedMessageIdDuplicateCases").GetInt32())); + foreach (var vector in cases) + { + Assert.That(vector.GetProperty("seesDuplicate").GetString(), Is.EqualTo("refuse")); + Assert.That(() => CultNetSchemaMessageSerialization.Deserialize(Convert.FromHexString(vector.GetProperty("wireHex").GetString()!)), + Throws.Exception, vector.GetProperty("name").GetString()); + } + } } } From e08d5c47f4cfe45e978c2ac5b8399b4cdd63b6b3 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:05:21 +0000 Subject: [PATCH 38/40] GameCult.Networking: a write_admitted frame with more than one messageId key is malformed Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetSchemaMessageSerialization.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index cb33b707..1e7bdd08 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -111,21 +111,24 @@ namespace GameCult.Networking { private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); - /// Throws unless every top-level messageId key of the frame is a valid write_admitted id; other keys are left to the serializer. + /// Throws unless the frame's one top-level messageId key is a valid write_admitted id (a repeated key is malformed); other keys are left to the serializer. public static void Require(byte[] payload) { var reader = new MessagePackReader(payload); var entries = reader.ReadMapHeader(); + var seen = false; for (var index = 0; index < entries; index++) { var key = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; if (key == "messageId" && reader.NextMessagePackType == MessagePackType.String) { + if (seen) throw new MessagePackSerializationException("cultnet.write_admitted.v0 carries messageId more than once."); + seen = true; string id; try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); - continue; // a repeated messageId key is checked too: the serializer keeps the last, so every one must be valid + continue; } if (key == null) reader.Skip(); From 27d9fbca9414cdaadb214095f66b9f643854f95a Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:07:11 +0000 Subject: [PATCH 39/40] write-receipt.md: where a received messageId decides, and the single-parser assumption it relies on Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 4f6cb7d1..5bb0a3e8 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,7 +12,9 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. -**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. Every `messageId` key of a frame must be valid. A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. A frame with more than one `messageId` key is malformed: a decoder that can see the duplicate (the C# and Rust decoders) refuses it; a decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses if that value is invalid, so no decoder echoes an invalid id (vectors `admittedMessageIdDuplicates` declare each policy and every runner asserts its own). A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. + +**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on every raw schema frame, write frames included; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. From 2aade7fdd32fd4ca5719c45d76365d545d546eba Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 9 Nov 2026 16:25:50 +0000 Subject: [PATCH 40/40] Docs and test comments: correct the Rust duplicate-key citation, say what the TS and Rust duplicate tests pin Co-Authored-By: Claude Sonnet 5.5 --- contracts/cultnet/interop/write-receipt.md | 2 +- packages/cultnet-rs/tests/write_receipt.rs | 3 ++- packages/cultnet-ts/test/write-receipt.test.ts | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index 5bb0a3e8..bfcbbb86 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -14,7 +14,7 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. A frame with more than one `messageId` key is malformed: a decoder that can see the duplicate (the C# and Rust decoders) refuses it; a decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses if that value is invalid, so no decoder echoes an invalid id (vectors `admittedMessageIdDuplicates` declare each policy and every runner asserts its own). A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. -**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on every raw schema frame, write frames included; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. +**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on write frames and on `write_admitted` (serde's `duplicate field messageId`, and `refuse_duplicate_keys` for the raw-list frames such as `document_put_raw`), though its `readable_message_id` names the first key when it names an unreadable frame; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index ee05bd72..3dd02a76 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -239,7 +239,8 @@ fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } -/// Vectors `admittedMessageIdDuplicates`: Rust decodes a map it can see into, so a repeated messageId key is refused whatever the values. +/// Vectors `admittedMessageIdDuplicates`: a repeated messageId key is refused whatever the values. This pins the outcome, not the +/// cause (today serde's `duplicate field messageId` in the typed decode of write_admitted). #[test] fn a_repeated_message_id_key_is_refused_whatever_the_values() { let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 1483fe31..23ef478e 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -325,6 +325,8 @@ test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned }); // TypeScript decodes the frame to an object, so it cannot see a duplicate key: it judges the last one (admittedMessageIdDuplicates). +// This pins @msgpack/msgpack's last-key behaviour plus the id check on the decoded object, not a first-key or duplicate path in +// cultnet-ts source: there is none, so a refuse-on-first mutation in src would not be killed. test("write_admitted messageId: a repeated key follows the last-key-wins policy", () => { const vectors = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); const section = vectors.admittedMessageIdDuplicates;