diff --git a/contracts/cultnet/interop/write-admitted-message-id-ranges.json b/contracts/cultnet/interop/write-admitted-message-id-ranges.json new file mode 100644 index 00000000..98ccdec0 --- /dev/null +++ b/contracts/cultnet/interop/write-admitted-message-id-ranges.json @@ -0,0 +1 @@ +{"generated":"generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit","unicodeVersion":"16.0.0","letterOrNumber":[[48,57],[65,90],[97,122],[170,170],[178,179],[181,181],[185,186],[188,190],[192,214],[216,246],[248,705],[710,721],[736,740],[748,748],[750,750],[880,884],[886,887],[890,893],[895,895],[902,902],[904,906],[908,908],[910,929],[931,1013],[1015,1153],[1162,1327],[1329,1366],[1369,1369],[1376,1416],[1488,1514],[1519,1522],[1568,1610],[1632,1641],[1646,1647],[1649,1747],[1749,1749],[1765,1766],[1774,1788],[1791,1791],[1808,1808],[1810,1839],[1869,1957],[1969,1969],[1984,2026],[2036,2037],[2042,2042],[2048,2069],[2074,2074],[2084,2084],[2088,2088],[2112,2136],[2144,2154],[2160,2183],[2185,2190],[2208,2249],[2308,2361],[2365,2365],[2384,2384],[2392,2401],[2406,2415],[2417,2432],[2437,2444],[2447,2448],[2451,2472],[2474,2480],[2482,2482],[2486,2489],[2493,2493],[2510,2510],[2524,2525],[2527,2529],[2534,2545],[2548,2553],[2556,2556],[2565,2570],[2575,2576],[2579,2600],[2602,2608],[2610,2611],[2613,2614],[2616,2617],[2649,2652],[2654,2654],[2662,2671],[2674,2676],[2693,2701],[2703,2705],[2707,2728],[2730,2736],[2738,2739],[2741,2745],[2749,2749],[2768,2768],[2784,2785],[2790,2799],[2809,2809],[2821,2828],[2831,2832],[2835,2856],[2858,2864],[2866,2867],[2869,2873],[2877,2877],[2908,2909],[2911,2913],[2918,2927],[2929,2935],[2947,2947],[2949,2954],[2958,2960],[2962,2965],[2969,2970],[2972,2972],[2974,2975],[2979,2980],[2984,2986],[2990,3001],[3024,3024],[3046,3058],[3077,3084],[3086,3088],[3090,3112],[3114,3129],[3133,3133],[3160,3162],[3165,3165],[3168,3169],[3174,3183],[3192,3198],[3200,3200],[3205,3212],[3214,3216],[3218,3240],[3242,3251],[3253,3257],[3261,3261],[3293,3294],[3296,3297],[3302,3311],[3313,3314],[3332,3340],[3342,3344],[3346,3386],[3389,3389],[3406,3406],[3412,3414],[3416,3425],[3430,3448],[3450,3455],[3461,3478],[3482,3505],[3507,3515],[3517,3517],[3520,3526],[3558,3567],[3585,3632],[3634,3635],[3648,3654],[3664,3673],[3713,3714],[3716,3716],[3718,3722],[3724,3747],[3749,3749],[3751,3760],[3762,3763],[3773,3773],[3776,3780],[3782,3782],[3792,3801],[3804,3807],[3840,3840],[3872,3891],[3904,3911],[3913,3948],[3976,3980],[4096,4138],[4159,4169],[4176,4181],[4186,4189],[4193,4193],[4197,4198],[4206,4208],[4213,4225],[4238,4238],[4240,4249],[4256,4293],[4295,4295],[4301,4301],[4304,4346],[4348,4680],[4682,4685],[4688,4694],[4696,4696],[4698,4701],[4704,4744],[4746,4749],[4752,4784],[4786,4789],[4792,4798],[4800,4800],[4802,4805],[4808,4822],[4824,4880],[4882,4885],[4888,4954],[4969,4988],[4992,5007],[5024,5109],[5112,5117],[5121,5740],[5743,5759],[5761,5786],[5792,5866],[5870,5880],[5888,5905],[5919,5937],[5952,5969],[5984,5996],[5998,6000],[6016,6067],[6103,6103],[6108,6108],[6112,6121],[6128,6137],[6160,6169],[6176,6264],[6272,6276],[6279,6312],[6314,6314],[6320,6389],[6400,6430],[6470,6509],[6512,6516],[6528,6571],[6576,6601],[6608,6618],[6656,6678],[6688,6740],[6784,6793],[6800,6809],[6823,6823],[6917,6963],[6981,6988],[6992,7001],[7043,7072],[7086,7141],[7168,7203],[7232,7241],[7245,7293],[7296,7306],[7312,7354],[7357,7359],[7401,7404],[7406,7411],[7413,7414],[7418,7418],[7424,7615],[7680,7957],[7960,7965],[7968,8005],[8008,8013],[8016,8023],[8025,8025],[8027,8027],[8029,8029],[8031,8061],[8064,8116],[8118,8124],[8126,8126],[8130,8132],[8134,8140],[8144,8147],[8150,8155],[8160,8172],[8178,8180],[8182,8188],[8304,8305],[8308,8313],[8319,8329],[8336,8348],[8450,8450],[8455,8455],[8458,8467],[8469,8469],[8473,8477],[8484,8484],[8486,8486],[8488,8488],[8490,8493],[8495,8505],[8508,8511],[8517,8521],[8526,8526],[8528,8585],[9312,9371],[9450,9471],[10102,10131],[11264,11492],[11499,11502],[11506,11507],[11517,11517],[11520,11557],[11559,11559],[11565,11565],[11568,11623],[11631,11631],[11648,11670],[11680,11686],[11688,11694],[11696,11702],[11704,11710],[11712,11718],[11720,11726],[11728,11734],[11736,11742],[11823,11823],[12293,12295],[12321,12329],[12337,12341],[12344,12348],[12353,12438],[12445,12447],[12449,12538],[12540,12543],[12549,12591],[12593,12686],[12690,12693],[12704,12735],[12784,12799],[12832,12841],[12872,12879],[12881,12895],[12928,12937],[12977,12991],[13312,19903],[19968,42124],[42192,42237],[42240,42508],[42512,42539],[42560,42606],[42623,42653],[42656,42735],[42775,42783],[42786,42888],[42891,42957],[42960,42961],[42963,42963],[42965,42972],[42994,43009],[43011,43013],[43015,43018],[43020,43042],[43056,43061],[43072,43123],[43138,43187],[43216,43225],[43250,43255],[43259,43259],[43261,43262],[43264,43301],[43312,43334],[43360,43388],[43396,43442],[43471,43481],[43488,43492],[43494,43518],[43520,43560],[43584,43586],[43588,43595],[43600,43609],[43616,43638],[43642,43642],[43646,43695],[43697,43697],[43701,43702],[43705,43709],[43712,43712],[43714,43714],[43739,43741],[43744,43754],[43762,43764],[43777,43782],[43785,43790],[43793,43798],[43808,43814],[43816,43822],[43824,43866],[43868,43881],[43888,44002],[44016,44025],[44032,55203],[55216,55238],[55243,55291],[63744,64109],[64112,64217],[64256,64262],[64275,64279],[64285,64285],[64287,64296],[64298,64310],[64312,64316],[64318,64318],[64320,64321],[64323,64324],[64326,64433],[64467,64829],[64848,64911],[64914,64967],[65008,65019],[65136,65140],[65142,65276],[65296,65305],[65313,65338],[65345,65370],[65382,65470],[65474,65479],[65482,65487],[65490,65495],[65498,65500],[65536,65547],[65549,65574],[65576,65594],[65596,65597],[65599,65613],[65616,65629],[65664,65786],[65799,65843],[65856,65912],[65930,65931],[66176,66204],[66208,66256],[66273,66299],[66304,66339],[66349,66378],[66384,66421],[66432,66461],[66464,66499],[66504,66511],[66513,66517],[66560,66717],[66720,66729],[66736,66771],[66776,66811],[66816,66855],[66864,66915],[66928,66938],[66940,66954],[66956,66962],[66964,66965],[66967,66977],[66979,66993],[66995,67001],[67003,67004],[67008,67059],[67072,67382],[67392,67413],[67424,67431],[67456,67461],[67463,67504],[67506,67514],[67584,67589],[67592,67592],[67594,67637],[67639,67640],[67644,67644],[67647,67669],[67672,67702],[67705,67742],[67751,67759],[67808,67826],[67828,67829],[67835,67867],[67872,67897],[67968,68023],[68028,68047],[68050,68096],[68112,68115],[68117,68119],[68121,68149],[68160,68168],[68192,68222],[68224,68255],[68288,68295],[68297,68324],[68331,68335],[68352,68405],[68416,68437],[68440,68466],[68472,68497],[68521,68527],[68608,68680],[68736,68786],[68800,68850],[68858,68899],[68912,68921],[68928,68965],[68975,68997],[69216,69246],[69248,69289],[69296,69297],[69314,69316],[69376,69415],[69424,69445],[69457,69460],[69488,69505],[69552,69579],[69600,69622],[69635,69687],[69714,69743],[69745,69746],[69749,69749],[69763,69807],[69840,69864],[69872,69881],[69891,69926],[69942,69951],[69956,69956],[69959,69959],[69968,70002],[70006,70006],[70019,70066],[70081,70084],[70096,70106],[70108,70108],[70113,70132],[70144,70161],[70163,70187],[70207,70208],[70272,70278],[70280,70280],[70282,70285],[70287,70301],[70303,70312],[70320,70366],[70384,70393],[70405,70412],[70415,70416],[70419,70440],[70442,70448],[70450,70451],[70453,70457],[70461,70461],[70480,70480],[70493,70497],[70528,70537],[70539,70539],[70542,70542],[70544,70581],[70583,70583],[70609,70609],[70611,70611],[70656,70708],[70727,70730],[70736,70745],[70751,70753],[70784,70831],[70852,70853],[70855,70855],[70864,70873],[71040,71086],[71128,71131],[71168,71215],[71236,71236],[71248,71257],[71296,71338],[71352,71352],[71360,71369],[71376,71395],[71424,71450],[71472,71483],[71488,71494],[71680,71723],[71840,71922],[71935,71942],[71945,71945],[71948,71955],[71957,71958],[71960,71983],[71999,71999],[72001,72001],[72016,72025],[72096,72103],[72106,72144],[72161,72161],[72163,72163],[72192,72192],[72203,72242],[72250,72250],[72272,72272],[72284,72329],[72349,72349],[72368,72440],[72640,72672],[72688,72697],[72704,72712],[72714,72750],[72768,72768],[72784,72812],[72818,72847],[72960,72966],[72968,72969],[72971,73008],[73030,73030],[73040,73049],[73056,73061],[73063,73064],[73066,73097],[73112,73112],[73120,73129],[73440,73458],[73474,73474],[73476,73488],[73490,73523],[73552,73561],[73648,73648],[73664,73684],[73728,74649],[74752,74862],[74880,75075],[77712,77808],[77824,78895],[78913,78918],[78944,82938],[82944,83526],[90368,90397],[90416,90425],[92160,92728],[92736,92766],[92768,92777],[92784,92862],[92864,92873],[92880,92909],[92928,92975],[92992,92995],[93008,93017],[93019,93025],[93027,93047],[93053,93071],[93504,93548],[93552,93561],[93760,93846],[93952,94026],[94032,94032],[94099,94111],[94176,94177],[94179,94179],[94208,100343],[100352,101589],[101631,101640],[110576,110579],[110581,110587],[110589,110590],[110592,110882],[110898,110898],[110928,110930],[110933,110933],[110948,110951],[110960,111355],[113664,113770],[113776,113788],[113792,113800],[113808,113817],[118000,118009],[119488,119507],[119520,119539],[119648,119672],[119808,119892],[119894,119964],[119966,119967],[119970,119970],[119973,119974],[119977,119980],[119982,119993],[119995,119995],[119997,120003],[120005,120069],[120071,120074],[120077,120084],[120086,120092],[120094,120121],[120123,120126],[120128,120132],[120134,120134],[120138,120144],[120146,120485],[120488,120512],[120514,120538],[120540,120570],[120572,120596],[120598,120628],[120630,120654],[120656,120686],[120688,120712],[120714,120744],[120746,120770],[120772,120779],[120782,120831],[122624,122654],[122661,122666],[122928,122989],[123136,123180],[123191,123197],[123200,123209],[123214,123214],[123536,123565],[123584,123627],[123632,123641],[124112,124139],[124144,124153],[124368,124397],[124400,124410],[124896,124902],[124904,124907],[124909,124910],[124912,124926],[124928,125124],[125127,125135],[125184,125251],[125259,125259],[125264,125273],[126065,126123],[126125,126127],[126129,126132],[126209,126253],[126255,126269],[126464,126467],[126469,126495],[126497,126498],[126500,126500],[126503,126503],[126505,126514],[126516,126519],[126521,126521],[126523,126523],[126530,126530],[126535,126535],[126537,126537],[126539,126539],[126541,126543],[126545,126546],[126548,126548],[126551,126551],[126553,126553],[126555,126555],[126557,126557],[126559,126559],[126561,126562],[126564,126564],[126567,126570],[126572,126578],[126580,126583],[126585,126588],[126590,126590],[126592,126601],[126603,126619],[126625,126627],[126629,126633],[126635,126651],[127232,127244],[130032,130041],[131072,173791],[173824,177977],[177984,178205],[178208,183969],[183984,191456],[191472,192093],[194560,195101],[196608,201546],[201552,205743]],"defaultIgnorable":[[173,173],[847,847],[1564,1564],[4447,4448],[6068,6069],[6155,6159],[8203,8207],[8234,8238],[8288,8303],[12644,12644],[65024,65039],[65279,65279],[65440,65440],[65520,65528],[113824,113827],[119155,119162],[917504,921599]]} diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index c7f47738..f99265ed 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -150,7 +150,7 @@ ] }, "admittedMessageId": { - "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string cannot hold: the lone surrogate U+D800 is not valid UTF-8, so the vector's messageId is a MessagePack str whose bytes are ED A0 80, and Rust, C#, Kotlin and Python build the frame from those bytes and must refuse it. Letters, numbers and Default_Ignorable are those of Unicode 16.0 in every runtime, from embedded tables (no platform category lookup), so a runtime's own Unicode version cannot change a result. Scope: write_admitted only; error.v1 is untouched.", + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. Every messageId key of a frame must be valid. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string or this file cannot hold (a lone surrogate is not valid UTF-8; the bytes are ED A0 80) or a frame that repeats a key. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses earlier; byte-level strictness is a property of the reader and is not required of a decoder. Scope: write_admitted only; error.v1 is untouched.", "cases": [ { "name": "refuse_braille_blank_2800", @@ -222,7 +222,7 @@ "D800" ], "expect": "refuse", - "why": "not a Unicode scalar value; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str, so the decoder must refuse the frame. Rust, C#, Kotlin and Python build the frame from wireHex; a runtime whose strings can hold a lone surrogate (TypeScript) may also build it from the code point", + "why": "a lone surrogate is not a Unicode scalar value and has no letter or number; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str. A reader that rejects invalid UTF-8 refuses the frame; one that substitutes or keeps the surrogate refuses the id", "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" }, { @@ -408,6 +408,122 @@ "expect": "refuse", "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0" }, + { + "name": "refuse_duplicate_message_id_second_is_default_ignorable", + "codePoints": ["0061"], + "expect": "refuse", + "why": "the frame carries messageId twice, \"a\" and then U+200B; every messageId key must be valid, and a reader that keeps the last one sees U+200B", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B" + }, + { + "name": "refuse_letter_then_lone_surrogate_D800", + "codePoints": ["0061"], + "expect": "refuse", + "why": "\"a\" followed by a lone surrogate: the id as decoded is not well-formed Unicode. A reader that rejects invalid UTF-8 refuses the frame earlier; one that substitutes or keeps the surrogate must still refuse the id. Byte-level strictness is a property of the reader, not required of the decoder", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A461EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "accept_astral_mathematical_bold_capital_a_1D400", + "codePoints": ["1D400"], + "expect": "accept", + "why": "an astral letter (Lu)" + }, + { + "name": "accept_astral_cjk_ext_b_20000", + "codePoints": ["20000"], + "expect": "accept", + "why": "an astral letter (Lo)" + }, + { + "name": "refuse_letter_then_default_ignorable_061C", + "codePoints": ["0061", "061C"], + "expect": "refuse", + "why": "U+061C is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B4", + "codePoints": ["0061", "17B4"], + "expect": "refuse", + "why": "U+17B4 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B5", + "codePoints": ["0061", "17B5"], + "expect": "refuse", + "why": "U+17B5 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202A", + "codePoints": ["0061", "202A"], + "expect": "refuse", + "why": "U+202A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202E", + "codePoints": ["0061", "202E"], + "expect": "refuse", + "why": "U+202E is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE00", + "codePoints": ["0061", "FE00"], + "expect": "refuse", + "why": "U+FE00 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE0F", + "codePoints": ["0061", "FE0F"], + "expect": "refuse", + "why": "U+FE0F is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF0", + "codePoints": ["0061", "FFF0"], + "expect": "refuse", + "why": "U+FFF0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF8", + "codePoints": ["0061", "FFF8"], + "expect": "refuse", + "why": "U+FFF8 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA0", + "codePoints": ["0061", "1BCA0"], + "expect": "refuse", + "why": "U+1BCA0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA3", + "codePoints": ["0061", "1BCA3"], + "expect": "refuse", + "why": "U+1BCA3 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D173", + "codePoints": ["0061", "1D173"], + "expect": "refuse", + "why": "U+1D173 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D17A", + "codePoints": ["0061", "1D17A"], + "expect": "refuse", + "why": "U+1D17A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0000", + "codePoints": ["0061", "E0000"], + "expect": "refuse", + "why": "U+E0000 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0FFF", + "codePoints": ["0061", "E0FFF"], + "expect": "refuse", + "why": "U+E0FFF is Default_Ignorable (an end of one of its ranges)" + }, { "name": "accept_abc", "codePoints": [ @@ -1524,7 +1640,7 @@ "schemaInvalid": 7, "contentHashCases": 4, "scenarioCases": 25, - "admittedMessageIdCases": 40 + "admittedMessageIdCases": 59 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index 8955c52d..35e0136f 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -68,6 +68,38 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every + // range, so deleting or narrowing any range fails here. + check("unicodeTableRanges") { + @Suppress("UNCHECKED_CAST") + val ranges = JsonReader(File(File(path).parentFile, "write-admitted-message-id-ranges.json").readText(Charsets.UTF_8)).readDocument() as Map + fun pairs(key: String) = list(ranges[key]).map { list(it).map { n -> (n as Long).toInt() } } + val letters = pairs("letterOrNumber") + val ignorable = pairs("defaultIgnorable") + val problems = mutableListOf() + if (letters.size <= 700 || ignorable.size != 17) problems += "ranges file holds ${letters.size} and ${ignorable.size} ranges" + fun accepted(id: String): Boolean = try { + decodeReceiptFrame(parseCultNetMessage(MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray())) + true + } catch (_: IOException) { false } + for ((low, high) in letters) for ((point, want) in listOf(low to true, high to true, low - 1 to false, high + 1 to false)) { + if (accepted(String(Character.toChars(point))) != want) problems += "U+${point.toString(16)} alone should be ${if (want) "accepted" else "refused"}" + } + for ((low, high) in ignorable) for ((point, want) in listOf(low to false, high to false, low - 1 to true, high + 1 to true)) { + if (accepted("a" + String(Character.toChars(point))) != want) problems += "a + U+${point.toString(16)} should be ${if (want) "accepted" else "refused"}" + } + problems + } + + // A document payload string with a lone surrogate (ED A0 80) is not a write_admitted id: the shared MessagePack reader keeps + // reading it. Strictness is scoped to the write_admitted frame, never to the reader. + check("documentPayloadStringWithLoneSurrogateStillDecodes") { + val text = MessagePackReader(unhex("A3EDA080")).readString() + if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string") + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 0c42a6af..fce0d850 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -94,6 +94,32 @@ class WriteReceiptVectorTests(unittest.TestCase): failures.append(f"{case['name']}: expected {case['expect']}") self.assertEqual(failures, []) + def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None: + # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and + # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and + # last point is refused and beside the points either side accepted. Deleting or narrowing any embedded range fails here. + ranges = json.loads((_INTEROP / "write-admitted-message-id-ranges.json").read_text(encoding="utf-8")) + self.assertGreater(len(ranges["letterOrNumber"]), 700) + self.assertEqual(len(ranges["defaultIgnorable"]), 17) + + def accepted(text: str) -> bool: + try: + parse_message(_pack(_admitted(messageId=text))) + return True + except ValueError: + return False + + failures = [] + for low, high in ranges["letterOrNumber"]: + for point, want in ((low, True), (high, True), (low - 1, False), (high + 1, False)): + if accepted(chr(point)) != want: + failures.append(f"U+{point:04X} alone should be {'accepted' if want else 'refused'}") + for low, high in ranges["defaultIgnorable"]: + for point, want in ((low, False), (high, False), (low - 1, True), (high + 1, True)): + if accepted("a" + chr(point)) != want: + failures.append(f"a + U+{point:04X} should be {'accepted' if want else 'refused'}") + self.assertEqual(failures, []) + def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: payload = None if case["payloadHex"] is None else bytes.fromhex(case["payloadHex"]) diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 8dccf996..10d7dcb8 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -201,3 +201,40 @@ fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() { } assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); } + +/// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +#[test] +fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-admitted-message-id-ranges.json"); + let ranges: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let letters = ranges["letterOrNumber"].as_array().unwrap(); + let ignorable = ranges["defaultIgnorable"].as_array().unwrap(); + assert!(letters.len() > 700); + assert_eq!(ignorable.len(), 17); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let accepted = |id: &str| { + let header = vec![0xA0 | id.len() as u8]; + let frame = [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat(); + decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0).is_ok() + }; + let point = |value: i64| char::from_u32(value as u32).unwrap().to_string(); + let mut failures = Vec::new(); + for range in letters { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, true), (high, true), (low - 1, false), (high + 1, false)] { + if accepted(&point(at)) != want { + failures.push(format!("U+{at:X} alone should be {}", if want { "accepted" } else { "refused" })); + } + } + } + for range in ignorable { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, false), (high, false), (low - 1, true), (high + 1, true)] { + if accepted(&format!("a{}", point(at))) != want { + failures.push(format!("a + U+{at:X} should be {}", if want { "accepted" } else { "refused" })); + } + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index c447367a..c654de1a 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -303,3 +303,23 @@ test("write_admitted messageId: every admittedMessageId vector is refused or acc } assert.deepEqual(failures, []); }); + +// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned at both ends", () => { + const ranges = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"), "utf8")); + assert.ok(ranges.letterOrNumber.length > 700); + assert.equal(ranges.defaultIgnorable.length, 17); + const accepted = (id: string) => { try { parseCultNetMessage(admitted(id)); return true; } catch { return false; } }; + const failures: string[] = []; + for (const [low, high] of ranges.letterOrNumber as [number, number][]) { + for (const [point, want] of [[low, true], [high, true], [low - 1, false], [high + 1, false]] as [number, boolean][]) { + if (accepted(String.fromCodePoint(point)) !== want) failures.push(`U+${point.toString(16)} alone should be ${want ? "accepted" : "refused"}`); + } + } + for (const [low, high] of ranges.defaultIgnorable as [number, number][]) { + for (const [point, want] of [[low, false], [high, false], [low - 1, true], [high + 1, true]] as [number, boolean][]) { + if (accepted("a" + String.fromCodePoint(point)) !== want) failures.push(`a + U+${point.toString(16)} should be ${want ? "accepted" : "refused"}`); + } + } + assert.deepEqual(failures, []); +}); diff --git a/scripts/gen-unicode-messageid-tables.py b/scripts/gen-unicode-messageid-tables.py index 96d1a205..4c119930 100644 --- a/scripts/gen-unicode-messageid-tables.py +++ b/scripts/gen-unicode-messageid-tables.py @@ -13,6 +13,7 @@ https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --chec any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only. """ import hashlib +import json import re import sys from pathlib import Path @@ -28,6 +29,9 @@ TARGETS = { "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs", "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs", } +# The independent copy the runtimes' tests probe (first, last, and the code point either side of every range), so that deleting +# or narrowing any range of any embedded table fails a test. +RANGES_JSON = ROOT / "contracts/cultnet/interop/write-admitted-message-id-ranges.json" def parse(path, wanted): @@ -95,8 +99,12 @@ def main(): digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16] header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit" stale = [] - for target, path in TARGETS.items(): - text = emit(target, letters, ignorable, header) + texts = {path: emit(target, letters, ignorable, header) for target, path in TARGETS.items()} + texts[RANGES_JSON] = json.dumps( + {"generated": header, "unicodeVersion": VERSION, "letterOrNumber": [list(r) for r in letters], "defaultIgnorable": [list(r) for r in ignorable]}, + separators=(",", ":"), + ) + "\n" + for path, text in texts.items(): if check: if not path.exists() or path.read_text(encoding="utf-8") != text: stale.append(str(path.relative_to(ROOT))) @@ -104,7 +112,7 @@ def main(): path.write_text(text, encoding="utf-8") if stale: sys.exit("stale or hand-edited: " + ", ".join(stale)) - print(f"{'checked' if check else 'wrote'} {len(TARGETS)} tables: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") + print(f"{'checked' if check else 'wrote'} {len(texts)} files: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") if __name__ == "__main__": diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs index f5cac528..a4f1475c 100644 --- a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -54,5 +54,38 @@ namespace GameCult.Networking.Tests Assert.That(decoded!.MessageId, Is.EqualTo(id)); } } + + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. + [Test] + public void EveryRangeOfTheUnicodeTablesIsPinnedAtBothEnds() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var ranges = JsonDocument.Parse(File.ReadAllText(Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"))).RootElement; + var letters = ranges.GetProperty("letterOrNumber").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + var ignorable = ranges.GetProperty("defaultIgnorable").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + Assert.That(letters.Length, Is.GreaterThan(700)); + Assert.That(ignorable.Length, Is.EqualTo(17)); + static bool Accepted(string id) + { + try + { + CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize( + new CultNetWriteAdmittedMessage { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" })); + return true; + } + catch (Exception) { return false; } + } + + var failures = new List(); + foreach (var (low, high) in letters) + foreach (var (point, want) in new[] { (low, true), (high, true), (low - 1, false), (high + 1, false) }) + if (Accepted(char.ConvertFromUtf32(point)) != want) failures.Add($"U+{point:X} alone should be {(want ? "accepted" : "refused")}"); + foreach (var (low, high) in ignorable) + foreach (var (point, want) in new[] { (low, false), (high, false), (low - 1, true), (high + 1, true) }) + if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}"); + Assert.That(failures, Is.Empty); + } } }