tests first: duplicate messageId, ill-formed id, scoped UTF-8 strictness, every table range pinned (failing)

Failing before the fix: C# 1 (duplicate messageId, second is U+200B), TypeScript 1 (a + lone surrogate),
Kotlin 1 (a document payload string with a lone surrogate no longer decodes). Python and Rust already pass
the new vectors. The range probes read the generated write-admitted-message-id-ranges.json and pass on the
correct tables; they exist to fail when any range is deleted or narrowed.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

contracts/cultnet/interop/write-receipt-vectors.json

@@ -150,7 +150,7 @@
     ]
   },
   "admittedMessageId": {
-    "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Letters, numbers and Default_Ignorable are those of Unicode 16.0 in every runtime, from embedded tables (no platform category lookup), so a runtime's own Unicode version cannot change a result. Scope: write_admitted only; error.v1 is untouched.",
+    "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. Every messageId key of a frame must be valid. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string or this file cannot hold (a lone surrogate is not valid UTF-8; the bytes are ED A0 80) or a frame that repeats a key. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses earlier; byte-level strictness is a property of the reader and is not required of a decoder. Scope: write_admitted only; error.v1 is untouched.",
     "cases": [
       {
         "name": "refuse_braille_blank_2800",
@@ -222,7 +222,7 @@
           "D800"
         ],
         "expect": "refuse",
-        "why": "not a Unicode scalar value; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str, so the decoder must refuse the frame. Rust, C#, Kotlin and Python build the frame from wireHex; a runtime whose strings can hold a lone surrogate (TypeScript) may also build it from the code point",
+        "why": "a lone surrogate is not a Unicode scalar value and has no letter or number; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str. A reader that rejects invalid UTF-8 refuses the frame; one that substitutes or keeps the surrogate refuses the id",
         "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"
       },
       {
@@ -408,6 +408,122 @@
         "expect": "refuse",
         "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0"
       },
+      {
+        "name": "refuse_duplicate_message_id_second_is_default_ignorable",
+        "codePoints": ["0061"],
+        "expect": "refuse",
+        "why": "the frame carries messageId twice, \"a\" and then U+200B; every messageId key must be valid, and a reader that keeps the last one sees U+200B",
+        "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B"
+      },
+      {
+        "name": "refuse_letter_then_lone_surrogate_D800",
+        "codePoints": ["0061"],
+        "expect": "refuse",
+        "why": "\"a\" followed by a lone surrogate: the id as decoded is not well-formed Unicode. A reader that rejects invalid UTF-8 refuses the frame earlier; one that substitutes or keeps the surrogate must still refuse the id. Byte-level strictness is a property of the reader, not required of the decoder",
+        "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A461EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"
+      },
+      {
+        "name": "accept_astral_mathematical_bold_capital_a_1D400",
+        "codePoints": ["1D400"],
+        "expect": "accept",
+        "why": "an astral letter (Lu)"
+      },
+      {
+        "name": "accept_astral_cjk_ext_b_20000",
+        "codePoints": ["20000"],
+        "expect": "accept",
+        "why": "an astral letter (Lo)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_061C",
+        "codePoints": ["0061", "061C"],
+        "expect": "refuse",
+        "why": "U+061C is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_17B4",
+        "codePoints": ["0061", "17B4"],
+        "expect": "refuse",
+        "why": "U+17B4 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_17B5",
+        "codePoints": ["0061", "17B5"],
+        "expect": "refuse",
+        "why": "U+17B5 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_202A",
+        "codePoints": ["0061", "202A"],
+        "expect": "refuse",
+        "why": "U+202A is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_202E",
+        "codePoints": ["0061", "202E"],
+        "expect": "refuse",
+        "why": "U+202E is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_FE00",
+        "codePoints": ["0061", "FE00"],
+        "expect": "refuse",
+        "why": "U+FE00 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_FE0F",
+        "codePoints": ["0061", "FE0F"],
+        "expect": "refuse",
+        "why": "U+FE0F is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_FFF0",
+        "codePoints": ["0061", "FFF0"],
+        "expect": "refuse",
+        "why": "U+FFF0 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_FFF8",
+        "codePoints": ["0061", "FFF8"],
+        "expect": "refuse",
+        "why": "U+FFF8 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_1BCA0",
+        "codePoints": ["0061", "1BCA0"],
+        "expect": "refuse",
+        "why": "U+1BCA0 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_1BCA3",
+        "codePoints": ["0061", "1BCA3"],
+        "expect": "refuse",
+        "why": "U+1BCA3 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_1D173",
+        "codePoints": ["0061", "1D173"],
+        "expect": "refuse",
+        "why": "U+1D173 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_1D17A",
+        "codePoints": ["0061", "1D17A"],
+        "expect": "refuse",
+        "why": "U+1D17A is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_E0000",
+        "codePoints": ["0061", "E0000"],
+        "expect": "refuse",
+        "why": "U+E0000 is Default_Ignorable (an end of one of its ranges)"
+      },
+      {
+        "name": "refuse_letter_then_default_ignorable_E0FFF",
+        "codePoints": ["0061", "E0FFF"],
+        "expect": "refuse",
+        "why": "U+E0FFF is Default_Ignorable (an end of one of its ranges)"
+      },
       {
         "name": "accept_abc",
         "codePoints": [
@@ -1524,7 +1640,7 @@
     "schemaInvalid": 7,
     "contentHashCases": 4,
     "scenarioCases": 25,
-    "admittedMessageIdCases": 40
+    "admittedMessageIdCases": 59
   },
   "contentHash": {
     "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase",

packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt

@@ -68,6 +68,38 @@ fun replayWriteReceiptVectors(path: String): Boolean {
         }
     }
 
+    // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every
+    // range, so deleting or narrowing any range fails here.
+    check("unicodeTableRanges") {
+        @Suppress("UNCHECKED_CAST")
+        val ranges = JsonReader(File(File(path).parentFile, "write-admitted-message-id-ranges.json").readText(Charsets.UTF_8)).readDocument() as Map<String, Any?>
+        fun pairs(key: String) = list(ranges[key]).map { list(it).map { n -> (n as Long).toInt() } }
+        val letters = pairs("letterOrNumber")
+        val ignorable = pairs("defaultIgnorable")
+        val problems = mutableListOf<String>()
+        if (letters.size <= 700 || ignorable.size != 17) problems += "ranges file holds ${letters.size} and ${ignorable.size} ranges"
+        fun accepted(id: String): Boolean = try {
+            decodeReceiptFrame(parseCultNetMessage(MessagePackWriter().value(linkedMapOf(
+                "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1",
+            )).toByteArray()))
+            true
+        } catch (_: IOException) { false }
+        for ((low, high) in letters) for ((point, want) in listOf(low to true, high to true, low - 1 to false, high + 1 to false)) {
+            if (accepted(String(Character.toChars(point))) != want) problems += "U+${point.toString(16)} alone should be ${if (want) "accepted" else "refused"}"
+        }
+        for ((low, high) in ignorable) for ((point, want) in listOf(low to false, high to false, low - 1 to true, high + 1 to true)) {
+            if (accepted("a" + String(Character.toChars(point))) != want) problems += "a + U+${point.toString(16)} should be ${if (want) "accepted" else "refused"}"
+        }
+        problems
+    }
+
+    // A document payload string with a lone surrogate (ED A0 80) is not a write_admitted id: the shared MessagePack reader keeps
+    // reading it. Strictness is scoped to the write_admitted frame, never to the reader.
+    check("documentPayloadStringWithLoneSurrogateStillDecodes") {
+        val text = MessagePackReader(unhex("A3EDA080")).readString()
+        if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string")
+    }
+
     for (case in list(vectors["schemaInvalid"])) {
         val c = obj(case)
         check("schemaInvalid/${c["name"]}") {

packages/cultnet-py/tests/test_write_receipt.py

@@ -94,6 +94,32 @@ class WriteReceiptVectorTests(unittest.TestCase):
                 failures.append(f"{case['name']}: expected {case['expect']}")
         self.assertEqual(failures, [])
 
+    def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None:
+        # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and
+        # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and
+        # last point is refused and beside the points either side accepted. Deleting or narrowing any embedded range fails here.
+        ranges = json.loads((_INTEROP / "write-admitted-message-id-ranges.json").read_text(encoding="utf-8"))
+        self.assertGreater(len(ranges["letterOrNumber"]), 700)
+        self.assertEqual(len(ranges["defaultIgnorable"]), 17)
+
+        def accepted(text: str) -> bool:
+            try:
+                parse_message(_pack(_admitted(messageId=text)))
+                return True
+            except ValueError:
+                return False
+
+        failures = []
+        for low, high in ranges["letterOrNumber"]:
+            for point, want in ((low, True), (high, True), (low - 1, False), (high + 1, False)):
+                if accepted(chr(point)) != want:
+                    failures.append(f"U+{point:04X} alone should be {'accepted' if want else 'refused'}")
+        for low, high in ranges["defaultIgnorable"]:
+            for point, want in ((low, False), (high, False), (low - 1, True), (high + 1, True)):
+                if accepted("a" + chr(point)) != want:
+                    failures.append(f"a + U+{point:04X} should be {'accepted' if want else 'refused'}")
+        self.assertEqual(failures, [])
+
     def test_content_hash_matches_every_vector(self) -> None:
         for case in _VECTORS["contentHash"]["cases"]:
             payload = None if case["payloadHex"] is None else bytes.fromhex(case["payloadHex"])

packages/cultnet-rs/tests/write_receipt.rs

@@ -201,3 +201,40 @@ fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() {
     }
     assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n"));
 }
+
+/// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range.
+#[test]
+fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() {
+    let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-admitted-message-id-ranges.json");
+    let ranges: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
+    let letters = ranges["letterOrNumber"].as_array().unwrap();
+    let ignorable = ranges["defaultIgnorable"].as_array().unwrap();
+    assert!(letters.len() > 700);
+    assert_eq!(ignorable.len(), 17);
+    let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31");
+    let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1;
+    let accepted = |id: &str| {
+        let header = vec![0xA0 | id.len() as u8];
+        let frame = [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat();
+        decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0).is_ok()
+    };
+    let point = |value: i64| char::from_u32(value as u32).unwrap().to_string();
+    let mut failures = Vec::new();
+    for range in letters {
+        let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap());
+        for (at, want) in [(low, true), (high, true), (low - 1, false), (high + 1, false)] {
+            if accepted(&point(at)) != want {
+                failures.push(format!("U+{at:X} alone should be {}", if want { "accepted" } else { "refused" }));
+            }
+        }
+    }
+    for range in ignorable {
+        let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap());
+        for (at, want) in [(low, false), (high, false), (low - 1, true), (high + 1, true)] {
+            if accepted(&format!("a{}", point(at))) != want {
+                failures.push(format!("a + U+{at:X} should be {}", if want { "accepted" } else { "refused" }));
+            }
+        }
+    }
+    assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n"));
+}

packages/cultnet-ts/test/write-receipt.test.ts

@@ -303,3 +303,23 @@ test("write_admitted messageId: every admittedMessageId vector is refused or acc
   }
   assert.deepEqual(failures, []);
 });
+
+// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range.
+test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned at both ends", () => {
+  const ranges = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"), "utf8"));
+  assert.ok(ranges.letterOrNumber.length > 700);
+  assert.equal(ranges.defaultIgnorable.length, 17);
+  const accepted = (id: string) => { try { parseCultNetMessage(admitted(id)); return true; } catch { return false; } };
+  const failures: string[] = [];
+  for (const [low, high] of ranges.letterOrNumber as [number, number][]) {
+    for (const [point, want] of [[low, true], [high, true], [low - 1, false], [high + 1, false]] as [number, boolean][]) {
+      if (accepted(String.fromCodePoint(point)) !== want) failures.push(`U+${point.toString(16)} alone should be ${want ? "accepted" : "refused"}`);
+    }
+  }
+  for (const [low, high] of ranges.defaultIgnorable as [number, number][]) {
+    for (const [point, want] of [[low, false], [high, false], [low - 1, true], [high + 1, true]] as [number, boolean][]) {
+      if (accepted("a" + String.fromCodePoint(point)) !== want) failures.push(`a + U+${point.toString(16)} should be ${want ? "accepted" : "refused"}`);
+    }
+  }
+  assert.deepEqual(failures, []);
+});

scripts/gen-unicode-messageid-tables.py

@@ -13,6 +13,7 @@ https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --chec
 any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only.
 """
 import hashlib
+import json
 import re
 import sys
 from pathlib import Path
@@ -28,6 +29,9 @@ TARGETS = {
     "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs",
     "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs",
 }
+# The independent copy the runtimes' tests probe (first, last, and the code point either side of every range), so that deleting
+# or narrowing any range of any embedded table fails a test.
+RANGES_JSON = ROOT / "contracts/cultnet/interop/write-admitted-message-id-ranges.json"
 
 
 def parse(path, wanted):
@@ -95,8 +99,12 @@ def main():
     digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16]
     header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit"
     stale = []
-    for target, path in TARGETS.items():
-        text = emit(target, letters, ignorable, header)
+    texts = {path: emit(target, letters, ignorable, header) for target, path in TARGETS.items()}
+    texts[RANGES_JSON] = json.dumps(
+        {"generated": header, "unicodeVersion": VERSION, "letterOrNumber": [list(r) for r in letters], "defaultIgnorable": [list(r) for r in ignorable]},
+        separators=(",", ":"),
+    ) + "\n"
+    for path, text in texts.items():
         if check:
             if not path.exists() or path.read_text(encoding="utf-8") != text:
                 stale.append(str(path.relative_to(ROOT)))
@@ -104,7 +112,7 @@ def main():
             path.write_text(text, encoding="utf-8")
     if stale:
         sys.exit("stale or hand-edited: " + ", ".join(stale))
-    print(f"{'checked' if check else 'wrote'} {len(TARGETS)} tables: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges")
+    print(f"{'checked' if check else 'wrote'} {len(texts)} files: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges")
 
 
 if __name__ == "__main__":

tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs

@@ -54,5 +54,38 @@ namespace GameCult.Networking.Tests
                 Assert.That(decoded!.MessageId, Is.EqualTo(id));
             }
         }
+
+        // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range.
+        [Test]
+        public void EveryRangeOfTheUnicodeTablesIsPinnedAtBothEnds()
+        {
+            var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
+            while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln")))
+                directory = directory.Parent;
+            var ranges = JsonDocument.Parse(File.ReadAllText(Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"))).RootElement;
+            var letters = ranges.GetProperty("letterOrNumber").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray();
+            var ignorable = ranges.GetProperty("defaultIgnorable").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray();
+            Assert.That(letters.Length, Is.GreaterThan(700));
+            Assert.That(ignorable.Length, Is.EqualTo(17));
+            static bool Accepted(string id)
+            {
+                try
+                {
+                    CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize(
+                        new CultNetWriteAdmittedMessage { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" }));
+                    return true;
+                }
+                catch (Exception) { return false; }
+            }
+
+            var failures = new List<string>();
+            foreach (var (low, high) in letters)
+                foreach (var (point, want) in new[] { (low, true), (high, true), (low - 1, false), (high + 1, false) })
+                    if (Accepted(char.ConvertFromUtf32(point)) != want) failures.Add($"U+{point:X} alone should be {(want ? "accepted" : "refused")}");
+            foreach (var (low, high) in ignorable)
+                foreach (var (point, want) in new[] { (low, false), (high, false), (low - 1, true), (high + 1, true) })
+                    if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}");
+            Assert.That(failures, Is.Empty);
+        }
     }
 }