diff --git a/contracts/cultnet/interop/write-admitted-message-id-ranges.json b/contracts/cultnet/interop/write-admitted-message-id-ranges.json new file mode 100644 index 00000000..98ccdec0 --- /dev/null +++ b/contracts/cultnet/interop/write-admitted-message-id-ranges.json @@ -0,0 +1 @@ +{"generated":"generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit","unicodeVersion":"16.0.0","letterOrNumber":[[48,57],[65,90],[97,122],[170,170],[178,179],[181,181],[185,186],[188,190],[192,214],[216,246],[248,705],[710,721],[736,740],[748,748],[750,750],[880,884],[886,887],[890,893],[895,895],[902,902],[904,906],[908,908],[910,929],[931,1013],[1015,1153],[1162,1327],[1329,1366],[1369,1369],[1376,1416],[1488,1514],[1519,1522],[1568,1610],[1632,1641],[1646,1647],[1649,1747],[1749,1749],[1765,1766],[1774,1788],[1791,1791],[1808,1808],[1810,1839],[1869,1957],[1969,1969],[1984,2026],[2036,2037],[2042,2042],[2048,2069],[2074,2074],[2084,2084],[2088,2088],[2112,2136],[2144,2154],[2160,2183],[2185,2190],[2208,2249],[2308,2361],[2365,2365],[2384,2384],[2392,2401],[2406,2415],[2417,2432],[2437,2444],[2447,2448],[2451,2472],[2474,2480],[2482,2482],[2486,2489],[2493,2493],[2510,2510],[2524,2525],[2527,2529],[2534,2545],[2548,2553],[2556,2556],[2565,2570],[2575,2576],[2579,2600],[2602,2608],[2610,2611],[2613,2614],[2616,2617],[2649,2652],[2654,2654],[2662,2671],[2674,2676],[2693,2701],[2703,2705],[2707,2728],[2730,2736],[2738,2739],[2741,2745],[2749,2749],[2768,2768],[2784,2785],[2790,2799],[2809,2809],[2821,2828],[2831,2832],[2835,2856],[2858,2864],[2866,2867],[2869,2873],[2877,2877],[2908,2909],[2911,2913],[2918,2927],[2929,2935],[2947,2947],[2949,2954],[2958,2960],[2962,2965],[2969,2970],[2972,2972],[2974,2975],[2979,2980],[2984,2986],[2990,3001],[3024,3024],[3046,3058],[3077,3084],[3086,3088],[3090,3112],[3114,3129],[3133,3133],[3160,3162],[3165,3165],[3168,3169],[3174,3183],[3192,3198],[3200,3200],[3205,3212],[3214,3216],[3218,3240],[3242,3251],[3253,3257],[3261,3261],[3293,3294],[3296,3297],[3302,3311],[3313,3314],[3332,3340],[3342,3344],[3346,3386],[3389,3389],[3406,3406],[3412,3414],[3416,3425],[3430,3448],[3450,3455],[3461,3478],[3482,3505],[3507,3515],[3517,3517],[3520,3526],[3558,3567],[3585,3632],[3634,3635],[3648,3654],[3664,3673],[3713,3714],[3716,3716],[3718,3722],[3724,3747],[3749,3749],[3751,3760],[3762,3763],[3773,3773],[3776,3780],[3782,3782],[3792,3801],[3804,3807],[3840,3840],[3872,3891],[3904,3911],[3913,3948],[3976,3980],[4096,4138],[4159,4169],[4176,4181],[4186,4189],[4193,4193],[4197,4198],[4206,4208],[4213,4225],[4238,4238],[4240,4249],[4256,4293],[4295,4295],[4301,4301],[4304,4346],[4348,4680],[4682,4685],[4688,4694],[4696,4696],[4698,4701],[4704,4744],[4746,4749],[4752,4784],[4786,4789],[4792,4798],[4800,4800],[4802,4805],[4808,4822],[4824,4880],[4882,4885],[4888,4954],[4969,4988],[4992,5007],[5024,5109],[5112,5117],[5121,5740],[5743,5759],[5761,5786],[5792,5866],[5870,5880],[5888,5905],[5919,5937],[5952,5969],[5984,5996],[5998,6000],[6016,6067],[6103,6103],[6108,6108],[6112,6121],[6128,6137],[6160,6169],[6176,6264],[6272,6276],[6279,6312],[6314,6314],[6320,6389],[6400,6430],[6470,6509],[6512,6516],[6528,6571],[6576,6601],[6608,6618],[6656,6678],[6688,6740],[6784,6793],[6800,6809],[6823,6823],[6917,6963],[6981,6988],[6992,7001],[7043,7072],[7086,7141],[7168,7203],[7232,7241],[7245,7293],[7296,7306],[7312,7354],[7357,7359],[7401,7404],[7406,7411],[7413,7414],[7418,7418],[7424,7615],[7680,7957],[7960,7965],[7968,8005],[8008,8013],[8016,8023],[8025,8025],[8027,8027],[8029,8029],[8031,8061],[8064,8116],[8118,8124],[8126,8126],[8130,8132],[8134,8140],[8144,8147],[8150,8155],[8160,8172],[8178,8180],[8182,8188],[8304,8305],[8308,8313],[8319,8329],[8336,8348],[8450,8450],[8455,8455],[8458,8467],[8469,8469],[8473,8477],[8484,8484],[8486,8486],[8488,8488],[8490,8493],[8495,8505],[8508,8511],[8517,8521],[8526,8526],[8528,8585],[9312,9371],[9450,9471],[10102,10131],[11264,11492],[11499,11502],[11506,11507],[11517,11517],[11520,11557],[11559,11559],[11565,11565],[11568,11623],[11631,11631],[11648,11670],[11680,11686],[11688,11694],[11696,11702],[11704,11710],[11712,11718],[11720,11726],[11728,11734],[11736,11742],[11823,11823],[12293,12295],[12321,12329],[12337,12341],[12344,12348],[12353,12438],[12445,12447],[12449,12538],[12540,12543],[12549,12591],[12593,12686],[12690,12693],[12704,12735],[12784,12799],[12832,12841],[12872,12879],[12881,12895],[12928,12937],[12977,12991],[13312,19903],[19968,42124],[42192,42237],[42240,42508],[42512,42539],[42560,42606],[42623,42653],[42656,42735],[42775,42783],[42786,42888],[42891,42957],[42960,42961],[42963,42963],[42965,42972],[42994,43009],[43011,43013],[43015,43018],[43020,43042],[43056,43061],[43072,43123],[43138,43187],[43216,43225],[43250,43255],[43259,43259],[43261,43262],[43264,43301],[43312,43334],[43360,43388],[43396,43442],[43471,43481],[43488,43492],[43494,43518],[43520,43560],[43584,43586],[43588,43595],[43600,43609],[43616,43638],[43642,43642],[43646,43695],[43697,43697],[43701,43702],[43705,43709],[43712,43712],[43714,43714],[43739,43741],[43744,43754],[43762,43764],[43777,43782],[43785,43790],[43793,43798],[43808,43814],[43816,43822],[43824,43866],[43868,43881],[43888,44002],[44016,44025],[44032,55203],[55216,55238],[55243,55291],[63744,64109],[64112,64217],[64256,64262],[64275,64279],[64285,64285],[64287,64296],[64298,64310],[64312,64316],[64318,64318],[64320,64321],[64323,64324],[64326,64433],[64467,64829],[64848,64911],[64914,64967],[65008,65019],[65136,65140],[65142,65276],[65296,65305],[65313,65338],[65345,65370],[65382,65470],[65474,65479],[65482,65487],[65490,65495],[65498,65500],[65536,65547],[65549,65574],[65576,65594],[65596,65597],[65599,65613],[65616,65629],[65664,65786],[65799,65843],[65856,65912],[65930,65931],[66176,66204],[66208,66256],[66273,66299],[66304,66339],[66349,66378],[66384,66421],[66432,66461],[66464,66499],[66504,66511],[66513,66517],[66560,66717],[66720,66729],[66736,66771],[66776,66811],[66816,66855],[66864,66915],[66928,66938],[66940,66954],[66956,66962],[66964,66965],[66967,66977],[66979,66993],[66995,67001],[67003,67004],[67008,67059],[67072,67382],[67392,67413],[67424,67431],[67456,67461],[67463,67504],[67506,67514],[67584,67589],[67592,67592],[67594,67637],[67639,67640],[67644,67644],[67647,67669],[67672,67702],[67705,67742],[67751,67759],[67808,67826],[67828,67829],[67835,67867],[67872,67897],[67968,68023],[68028,68047],[68050,68096],[68112,68115],[68117,68119],[68121,68149],[68160,68168],[68192,68222],[68224,68255],[68288,68295],[68297,68324],[68331,68335],[68352,68405],[68416,68437],[68440,68466],[68472,68497],[68521,68527],[68608,68680],[68736,68786],[68800,68850],[68858,68899],[68912,68921],[68928,68965],[68975,68997],[69216,69246],[69248,69289],[69296,69297],[69314,69316],[69376,69415],[69424,69445],[69457,69460],[69488,69505],[69552,69579],[69600,69622],[69635,69687],[69714,69743],[69745,69746],[69749,69749],[69763,69807],[69840,69864],[69872,69881],[69891,69926],[69942,69951],[69956,69956],[69959,69959],[69968,70002],[70006,70006],[70019,70066],[70081,70084],[70096,70106],[70108,70108],[70113,70132],[70144,70161],[70163,70187],[70207,70208],[70272,70278],[70280,70280],[70282,70285],[70287,70301],[70303,70312],[70320,70366],[70384,70393],[70405,70412],[70415,70416],[70419,70440],[70442,70448],[70450,70451],[70453,70457],[70461,70461],[70480,70480],[70493,70497],[70528,70537],[70539,70539],[70542,70542],[70544,70581],[70583,70583],[70609,70609],[70611,70611],[70656,70708],[70727,70730],[70736,70745],[70751,70753],[70784,70831],[70852,70853],[70855,70855],[70864,70873],[71040,71086],[71128,71131],[71168,71215],[71236,71236],[71248,71257],[71296,71338],[71352,71352],[71360,71369],[71376,71395],[71424,71450],[71472,71483],[71488,71494],[71680,71723],[71840,71922],[71935,71942],[71945,71945],[71948,71955],[71957,71958],[71960,71983],[71999,71999],[72001,72001],[72016,72025],[72096,72103],[72106,72144],[72161,72161],[72163,72163],[72192,72192],[72203,72242],[72250,72250],[72272,72272],[72284,72329],[72349,72349],[72368,72440],[72640,72672],[72688,72697],[72704,72712],[72714,72750],[72768,72768],[72784,72812],[72818,72847],[72960,72966],[72968,72969],[72971,73008],[73030,73030],[73040,73049],[73056,73061],[73063,73064],[73066,73097],[73112,73112],[73120,73129],[73440,73458],[73474,73474],[73476,73488],[73490,73523],[73552,73561],[73648,73648],[73664,73684],[73728,74649],[74752,74862],[74880,75075],[77712,77808],[77824,78895],[78913,78918],[78944,82938],[82944,83526],[90368,90397],[90416,90425],[92160,92728],[92736,92766],[92768,92777],[92784,92862],[92864,92873],[92880,92909],[92928,92975],[92992,92995],[93008,93017],[93019,93025],[93027,93047],[93053,93071],[93504,93548],[93552,93561],[93760,93846],[93952,94026],[94032,94032],[94099,94111],[94176,94177],[94179,94179],[94208,100343],[100352,101589],[101631,101640],[110576,110579],[110581,110587],[110589,110590],[110592,110882],[110898,110898],[110928,110930],[110933,110933],[110948,110951],[110960,111355],[113664,113770],[113776,113788],[113792,113800],[113808,113817],[118000,118009],[119488,119507],[119520,119539],[119648,119672],[119808,119892],[119894,119964],[119966,119967],[119970,119970],[119973,119974],[119977,119980],[119982,119993],[119995,119995],[119997,120003],[120005,120069],[120071,120074],[120077,120084],[120086,120092],[120094,120121],[120123,120126],[120128,120132],[120134,120134],[120138,120144],[120146,120485],[120488,120512],[120514,120538],[120540,120570],[120572,120596],[120598,120628],[120630,120654],[120656,120686],[120688,120712],[120714,120744],[120746,120770],[120772,120779],[120782,120831],[122624,122654],[122661,122666],[122928,122989],[123136,123180],[123191,123197],[123200,123209],[123214,123214],[123536,123565],[123584,123627],[123632,123641],[124112,124139],[124144,124153],[124368,124397],[124400,124410],[124896,124902],[124904,124907],[124909,124910],[124912,124926],[124928,125124],[125127,125135],[125184,125251],[125259,125259],[125264,125273],[126065,126123],[126125,126127],[126129,126132],[126209,126253],[126255,126269],[126464,126467],[126469,126495],[126497,126498],[126500,126500],[126503,126503],[126505,126514],[126516,126519],[126521,126521],[126523,126523],[126530,126530],[126535,126535],[126537,126537],[126539,126539],[126541,126543],[126545,126546],[126548,126548],[126551,126551],[126553,126553],[126555,126555],[126557,126557],[126559,126559],[126561,126562],[126564,126564],[126567,126570],[126572,126578],[126580,126583],[126585,126588],[126590,126590],[126592,126601],[126603,126619],[126625,126627],[126629,126633],[126635,126651],[127232,127244],[130032,130041],[131072,173791],[173824,177977],[177984,178205],[178208,183969],[183984,191456],[191472,192093],[194560,195101],[196608,201546],[201552,205743]],"defaultIgnorable":[[173,173],[847,847],[1564,1564],[4447,4448],[6068,6069],[6155,6159],[8203,8207],[8234,8238],[8288,8303],[12644,12644],[65024,65039],[65279,65279],[65440,65440],[65520,65528],[113824,113827],[119155,119162],[917504,921599]]} diff --git a/contracts/cultnet/interop/write-receipt-vectors.json b/contracts/cultnet/interop/write-receipt-vectors.json index 397fbad6..e95c3180 100644 --- a/contracts/cultnet/interop/write-receipt-vectors.json +++ b/contracts/cultnet/interop/write-receipt-vectors.json @@ -149,6 +149,488 @@ { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" } ] }, + "admittedMessageId": { + "semantics": "Decoder-side rule for a `cultnet.write_admitted.v0` frame's `messageId` (Heimdall's ruling, write-receipt.md). Valid only if it contains at least one Unicode letter (\\p{L}) or number (\\p{N}) AND no Default_Ignorable_Code_Point (DerivedCoreProperties). The compare is RAW: the id is echoed byte for byte, so a decoder neither trims nor normalizes. Each case's id is given as `codePoints` (hex, in order), repeated `repeat` times (default 1), so no lone surrogate appears in this file. Every messageId key of a frame must be valid. A runner builds the frame {schemaVersion write_admitted.v0, messageId, operation put, schemaId note.v0, recordKey note:1} with its own encoder and decodes it with the write_admitted decoder; `refuse` means the decoder refuses the frame the way it refuses an empty or ill-typed messageId, `accept` means it decodes and the id comes back identical. A case with `wireHex` carries the whole frame in MessagePack hex, for an id a language string or this file cannot hold (a lone surrogate is not valid UTF-8; the bytes are ED A0 80) or a frame that repeats a key. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses earlier; byte-level strictness is a property of the reader and is not required of a decoder. Scope: write_admitted only; error.v1 is untouched.", + "cases": [ + { + "name": "refuse_braille_blank_2800", + "codePoints": [ + "2800" + ], + "expect": "refuse", + "why": "U+2800 is Default_Ignorable and not a letter or number" + }, + { + "name": "refuse_hangul_filler_3164", + "codePoints": [ + "3164" + ], + "expect": "refuse", + "why": "U+3164 Default_Ignorable (a letter by category Lo, so DI decides)" + }, + { + "name": "refuse_hangul_choseong_filler_115F", + "codePoints": [ + "115F" + ], + "expect": "refuse", + "why": "U+115F Default_Ignorable (category Lo)" + }, + { + "name": "refuse_halfwidth_hangul_filler_FFA0", + "codePoints": [ + "FFA0" + ], + "expect": "refuse", + "why": "U+FFA0 Default_Ignorable (category Lo)" + }, + { + "name": "refuse_combining_grapheme_joiner_034F", + "codePoints": [ + "034F" + ], + "expect": "refuse", + "why": "U+034F Default_Ignorable" + }, + { + "name": "refuse_combining_mark_alone_0300", + "codePoints": [ + "0300" + ], + "expect": "refuse", + "why": "a combining mark is not a letter or number" + }, + { + "name": "refuse_private_use_E000", + "codePoints": [ + "E000" + ], + "expect": "refuse", + "why": "private use is neither a letter nor a number" + }, + { + "name": "refuse_unassigned_0378", + "codePoints": [ + "0378" + ], + "expect": "refuse", + "why": "unassigned since Unicode 1.1, neither a letter nor a number" + }, + { + "name": "refuse_lone_surrogate_D800", + "codePoints": [ + "D800" + ], + "expect": "refuse", + "why": "a lone surrogate is not a Unicode scalar value and has no letter or number; on the wire it is the invalid UTF-8 bytes ED A0 80 in a str. A reader that rejects invalid UTF-8 refuses the frame; one that substitutes or keeps the surrogate refuses the id", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "refuse_control_nul_0000", + "codePoints": [ + "0000" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_bel_0007", + "codePoints": [ + "0007" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_unit_separator_001F", + "codePoints": [ + "001F" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_delete_007F", + "codePoints": [ + "007F" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_control_next_line_0085", + "codePoints": [ + "0085" + ], + "expect": "refuse", + "why": "a control character is neither a letter nor a number" + }, + { + "name": "refuse_empty", + "codePoints": [], + "expect": "refuse", + "why": "empty" + }, + { + "name": "refuse_whitespace_only_space_0020", + "codePoints": [ + "0020" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_whitespace_only_nbsp_00A0", + "codePoints": [ + "00A0" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_whitespace_only_ideographic_space_3000", + "codePoints": [ + "3000" + ], + "expect": "refuse", + "why": "white space only" + }, + { + "name": "refuse_letter_zwsp_letter", + "codePoints": [ + "0061", + "200B", + "0062" + ], + "expect": "refuse", + "why": "a Default_Ignorable (U+200B) beside letters" + }, + { + "name": "refuse_letter_cgj", + "codePoints": [ + "0061", + "034F" + ], + "expect": "refuse", + "why": "a Default_Ignorable (U+034F) after a letter" + }, + { + "name": "refuse_letter_soft_hyphen", + "codePoints": [ + "0061", + "00AD" + ], + "expect": "refuse", + "why": "U+00AD is Default_Ignorable" + }, + { + "name": "refuse_letter_zwj", + "codePoints": [ + "0061", + "200D" + ], + "expect": "refuse", + "why": "U+200D is Default_Ignorable" + }, + { + "name": "refuse_digit_mongolian_vowel_separator", + "codePoints": [ + "0031", + "180E" + ], + "expect": "refuse", + "why": "U+180E is Default_Ignorable since Unicode 6.3" + }, + { + "name": "refuse_digit_word_joiner_digit", + "codePoints": [ + "0031", + "2060", + "0032" + ], + "expect": "refuse", + "why": "U+2060 is Default_Ignorable" + }, + { + "name": "refuse_bom_before_letter", + "codePoints": [ + "FEFF", + "0061" + ], + "expect": "refuse", + "why": "U+FEFF is Default_Ignorable; JavaScript trim() would strip it and accept the rest, but the compare is raw" + }, + { + "name": "accept_unicode16_gurung_khema_letter_16100", + "codePoints": [ + "16100" + ], + "expect": "accept", + "why": "U+16100 GURUNG KHEMA LETTER A is Lo and new in Unicode 16.0 (unassigned in 15.1); the rule is pinned to 16.0 in every runtime" + }, + { + "name": "accept_unicode16_garay_digit_zero_10D40", + "codePoints": [ + "10D40" + ], + "expect": "accept", + "why": "U+10D40 GARAY DIGIT ZERO is Nd and new in Unicode 16.0" + }, + { + "name": "accept_unicode15_1_cjk_ext_i_2EBF0", + "codePoints": [ + "2EBF0" + ], + "expect": "accept", + "why": "U+2EBF0, first of CJK Unified Ideographs Extension I, is Lo and new in Unicode 15.1" + }, + { + "name": "accept_last_cjk_ext_i_2EE5D", + "codePoints": [ + "2EE5D" + ], + "expect": "accept", + "why": "U+2EE5D is the last assigned code point of Extension I (Lo)" + }, + { + "name": "refuse_after_cjk_ext_i_2EE5E", + "codePoints": [ + "2EE5E" + ], + "expect": "refuse", + "why": "U+2EE5E is unassigned in Unicode 16.0, so neither a letter nor a number" + }, + { + "name": "refuse_unassigned_in_16_0_10D90", + "codePoints": [ + "10D90" + ], + "expect": "refuse", + "why": "U+10D90 follows the Garay block and is unassigned in Unicode 16.0" + }, + { + "name": "refuse_duplicate_message_id_second_is_default_ignorable", + "codePoints": ["0061"], + "expect": "refuse", + "why": "the frame carries messageId twice, \"a\" and then U+200B; every messageId key must be valid, and a reader that keeps the last one sees U+200B", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B" + }, + { + "name": "refuse_letter_then_lone_surrogate_D800", + "codePoints": ["0061"], + "expect": "refuse", + "why": "\"a\" followed by a lone surrogate: the id as decoded is not well-formed Unicode. A reader that rejects invalid UTF-8 refuses the frame earlier; one that substitutes or keeps the surrogate must still refuse the id. Byte-level strictness is a property of the reader, not required of the decoder", + "wireHex": "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A461EDA080A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31" + }, + { + "name": "accept_astral_mathematical_bold_capital_a_1D400", + "codePoints": ["1D400"], + "expect": "accept", + "why": "an astral letter (Lu)" + }, + { + "name": "accept_astral_cjk_ext_b_20000", + "codePoints": ["20000"], + "expect": "accept", + "why": "an astral letter (Lo)" + }, + { + "name": "refuse_letter_then_default_ignorable_061C", + "codePoints": ["0061", "061C"], + "expect": "refuse", + "why": "U+061C is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B4", + "codePoints": ["0061", "17B4"], + "expect": "refuse", + "why": "U+17B4 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_17B5", + "codePoints": ["0061", "17B5"], + "expect": "refuse", + "why": "U+17B5 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202A", + "codePoints": ["0061", "202A"], + "expect": "refuse", + "why": "U+202A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_202E", + "codePoints": ["0061", "202E"], + "expect": "refuse", + "why": "U+202E is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE00", + "codePoints": ["0061", "FE00"], + "expect": "refuse", + "why": "U+FE00 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FE0F", + "codePoints": ["0061", "FE0F"], + "expect": "refuse", + "why": "U+FE0F is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF0", + "codePoints": ["0061", "FFF0"], + "expect": "refuse", + "why": "U+FFF0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_FFF8", + "codePoints": ["0061", "FFF8"], + "expect": "refuse", + "why": "U+FFF8 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA0", + "codePoints": ["0061", "1BCA0"], + "expect": "refuse", + "why": "U+1BCA0 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1BCA3", + "codePoints": ["0061", "1BCA3"], + "expect": "refuse", + "why": "U+1BCA3 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D173", + "codePoints": ["0061", "1D173"], + "expect": "refuse", + "why": "U+1D173 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_1D17A", + "codePoints": ["0061", "1D17A"], + "expect": "refuse", + "why": "U+1D17A is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0000", + "codePoints": ["0061", "E0000"], + "expect": "refuse", + "why": "U+E0000 is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "refuse_letter_then_default_ignorable_E0FFF", + "codePoints": ["0061", "E0FFF"], + "expect": "refuse", + "why": "U+E0FFF is Default_Ignorable (an end of one of its ranges)" + }, + { + "name": "accept_abc", + "codePoints": [ + "0061", + "0062", + "0063" + ], + "expect": "accept", + "why": "letters" + }, + { + "name": "accept_digit_1", + "codePoints": [ + "0031" + ], + "expect": "accept", + "why": "a number" + }, + { + "name": "accept_padded_a_raw_compare", + "codePoints": [ + "0020", + "0061", + "0020" + ], + "expect": "accept", + "why": "raw compare: the id is echoed byte for byte, never trimmed, so the padding stays and the letter makes it valid" + }, + { + "name": "accept_e_acute_precomposed", + "codePoints": [ + "00E9" + ], + "expect": "accept", + "why": "a letter" + }, + { + "name": "accept_arabic_indic_digit_three", + "codePoints": [ + "0663" + ], + "expect": "accept", + "why": "category Nd, a number" + }, + { + "name": "accept_cjk_zhong", + "codePoints": [ + "4E2D" + ], + "expect": "accept", + "why": "a letter (Lo, not Default_Ignorable)" + }, + { + "name": "accept_letter_then_combining_acute", + "codePoints": [ + "0065", + "0301" + ], + "expect": "accept", + "why": "a combining mark after a letter is allowed" + }, + { + "name": "accept_letter_nul_letter", + "codePoints": [ + "0061", + "0000", + "0062" + ], + "expect": "accept", + "why": "U+0000 is not Default_Ignorable and a letter is present, so the rule accepts; raw compare keeps the NUL" + }, + { + "name": "accept_256_ascii_letters", + "codePoints": [ + "0061" + ], + "expect": "accept", + "why": "256 bytes is the size limit and is allowed", + "repeat": 256 + } + ] + }, + "admittedMessageIdDuplicates": { + "semantics": "A frame with more than one messageId key is malformed. A decoder that can see the duplicate (C#, Rust: raw MessagePack reader or ordered pairs) refuses it, whatever the values (`seesDuplicate`). A decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses exactly when that value is invalid (`lastKeyWins`); where it accepts it echoes the last key's id (`lastKeyWinsEcho`). Neither behaviour ever echoes an invalid id. Each runtime declares which policy it has and its runner asserts that policy's column, so a change of a decoder's key policy fails a named test. The first key is ZWSP (U+200B, Default_Ignorable) in `invalid_then_valid`; the second key is the last entry of the map. Frames are MessagePack hex.", + "cases": [ + { + "name": "invalid_then_valid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A3E2808BA96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A161", + "seesDuplicate": "refuse", + "lastKeyWins": "accept", + "lastKeyWinsEcho": "a" + }, + { + "name": "valid_then_invalid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A161A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A3E2808B", + "seesDuplicate": "refuse", + "lastKeyWins": "refuse" + }, + { + "name": "valid_then_valid", + "wireHex": "86AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A161A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31A96D6573736167654964A162", + "seesDuplicate": "refuse", + "lastKeyWins": "accept", + "lastKeyWinsEcho": "b" + } + ] + }, "schemaInvalid": [ { "name": "write_admitted_with_an_error_key", @@ -1182,7 +1664,9 @@ "wire": 7, "schemaInvalid": 7, "contentHashCases": 4, - "scenarioCases": 25 + "scenarioCases": 25, + "admittedMessageIdCases": 59, + "admittedMessageIdDuplicateCases": 3 }, "contentHash": { "definition": "sha256(msgpack([operation, schemaId, recordKey, payload])), payload a msgpack bin for a put and nil for a delete; hex lowercase", diff --git a/contracts/cultnet/interop/write-receipt.md b/contracts/cultnet/interop/write-receipt.md index dc138452..bfcbbb86 100644 --- a/contracts/cultnet/interop/write-receipt.md +++ b/contracts/cultnet/interop/write-receipt.md @@ -12,6 +12,10 @@ A put or delete today has two answers on the wire: nothing (admitted, or the ser **`cultnet.write_admitted.v0`**: `{ schemaVersion, messageId, operation: "put"|"delete", schemaId, recordKey }`, every key written, none nil. Sent once a put or delete is admitted. The id echoes the write's `messageId` byte for byte; `operation`, `schemaId`, `recordKey` echo the write. +**A `write_admitted` `messageId` is valid only if it contains at least one Unicode letter (`\p{L}`) or number (`\p{N}`) and no Default_Ignorable_Code_Point (DerivedCoreProperties).** The compare is raw: the id is echoed byte for byte, so a decoder never trims or normalizes it (`" a "` is valid; U+2800 is not, and neither is `a` followed by U+200B). Unlike the Required fields rule, this is not a White_Space test. The letters, numbers and Default_Ignorable code points are those of **Unicode 16.0** in every runtime, taken from tables embedded in each runtime and generated by `scripts/gen-unicode-messageid-tables.py` from `DerivedGeneralCategory.txt` (Lu Ll Lt Lm Lo Nd Nl No) and `DerivedCoreProperties.txt`; no runtime asks its platform's Unicode data, so a runtime's own Unicode version cannot change a result. Bumping the version is one coordinated change: regenerate the five tables and update the vectors that pin the version in the same commit. The rule is on the decoded string, which must be well-formed Unicode: a lone surrogate refuses. A runtime whose MessagePack reader rejects invalid UTF-8 refuses the frame earlier; byte-level strictness is a property of the reader and is not required of a decoder. A frame with more than one `messageId` key is malformed: a decoder that can see the duplicate (the C# and Rust decoders) refuses it; a decoder that collapses the map (Python, TypeScript, Kotlin) judges the value it ends up with, the last key, and refuses if that value is invalid, so no decoder echoes an invalid id (vectors `admittedMessageIdDuplicates` declare each policy and every runner asserts its own). A decoder refuses an invalid id the way it refuses an empty or ill-typed one; the rule is for `write_admitted` only and `error.v1` is unchanged. + +**Where a received `messageId` decides, and what that relies on.** Beyond echoing it, a received `messageId` is used in exactly two places. On a server, the remembered-write key `(actor, messageId)` with the content hash, the per-session limit and the replay of rule 5 are taken from the id of the write frame the server itself parsed. On a client, the write tracker attributes a `write_admitted` or `error.v1` to a write by equality with the id of a write that client itself sent (a frame naming an id not in flight settles nothing), and attributes an id-less error by the count of unanswered writes, not by any id. Both rely on the receiver being the only parser of that frame: a decoder's `messageId` on a frame that carries more than one `messageId` key is not trusted for any authorization, dedupe, replay or ordering decision, and the map-collapsing runtimes (Python, TypeScript, Kotlin, and C#'s typed deserializer on write frames) cannot see the duplicate. If a frame is ever parsed by two runtimes in a chain (a relay, proxy or forwarder that re-reads it), or a consumer needs `messageId` for authorization, dedupe, replay or ordering beyond echo and equality with a self-sent id, all runtimes must first refuse a duplicate `messageId` key (a spec change plus byte-level duplicate detection in the runtimes that decode to a map) before that use lands. Today Rust refuses a repeated key on write frames and on `write_admitted` (serde's `duplicate field messageId`, and `refuse_duplicate_keys` for the raw-list frames such as `document_put_raw`), though its `readable_message_id` names the first key when it names an unreadable frame; C# does not on write frames (a duplicate-key write is deduped on its last id, while the id it names in a refusal for an unreadable frame is read from the first key), and the Python server (`cultmesh_py/server.py`) only echoes the id while the TypeScript and Kotlin code matches it by equality with a request it sent. + **`cultnet.error.v1`**: `cultnet.error.v0` plus a required `messageId` (string or nil). The code enum is v0's plus `limit_reached` (see Remembered-write limits); `cultnet.error.v1.schema.json` is the authority for the closed set. Sent in place of `error.v0` to a receipt-capable connection. `messageId` is the id of the request the error answers; nil only when the frame was unreadable or carried no id. Why a new error version and not an extra key on v0: the v0 schema is closed (`additionalProperties: false`) and the TypeScript decoder (Ajv over the closed schemas) rejects unknown keys and unknown `schemaVersion`s, and `parseCultNetMessage` throws on an unknown version; Rust's decoder errors on an unknown version too. Unknown-key tolerance of the Rust, Python and Kotlin error decoders was not checked, so none is assumed, so a surprise key or frame breaks an old client's whole receive path. Old clients are therefore never sent either new frame. This reverses the "old clients ignore it" assumption, which is false for today's decoders. @@ -65,7 +69,7 @@ Those clients complete *successfully* when the window ends with no refusal. Unde ## Vectors -`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. +`write-receipt-vectors.json`: `constants` (the named windows and retention), `contentHash` (worked dedupe hashes), `dedupe` (server replay cases for rule 5), `wire` (MessagePack hex plus the decoded message for each new frame; the bytes are spec-authored with msgpack-python and every runtime's first conformance PR re-captures them with its own encoder and must match), `schemaInvalid` (frames that must be refused by a validator, each with why), and `blankFields` (server input cases for the Required fields rule: which document fields make a put unreadable, which are accepted), `admittedMessageId` (the `write_admitted` id rule above: code-point lists, each marked refuse or accept; a lone surrogate, and a frame that repeats the key, are carried as whole frames in `wireHex`, because a lone surrogate is not valid UTF-8 and only some runtimes' strings can hold it; `write-admitted-message-id-ranges.json` lists every range of the embedded tables, and each runtime probes the first and last point of each range and the points either side), `scenarios` (event replays with the outcome each write must settle as; the clock is abstract; each case names its window constant). A conformant client passes every scenario; a conformant server's answers for the same inputs satisfy server rules 1 to 7. ## Conformance status diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt index 4a06d7f5..72c1d41d 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/CultMesh.kt @@ -19,6 +19,7 @@ import java.net.Socket import java.net.SocketTimeoutException import java.net.URI import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction import java.nio.charset.StandardCharsets import java.security.SecureRandom import java.security.MessageDigest @@ -2364,6 +2365,9 @@ fun parseCultNetMessage(payload: ByteArray): CultNetMessage { if (schemaVersion !is String || schemaVersion.isBlank()) { throw IOException("CultNet schema-v0 messages must declare schemaVersion") } + // A write_admitted id is echoed byte for byte, so that frame alone is read again with invalid UTF-8 refused; document payload + // strings elsewhere keep decoding as before. + if (schemaVersion == WRITE_ADMITTED_VERSION) MessagePackReader(payload, strictUtf8 = true).readAny() val body = linkedMapOf() for ((key, value) in decoded) { if (key !is String) throw IOException("CultNet schema-v0 message keys must be strings") @@ -6151,7 +6155,8 @@ private fun requireSchemaResponse(message: CultNetMessage?, expectedSchemaVersio return response } -class MessagePackReader(payload: ByteArray) { +/** [strictUtf8] makes a str whose bytes are not UTF-8 throw instead of becoming U+FFFD; only the write_admitted frame asks for it. */ +class MessagePackReader(payload: ByteArray, private val strictUtf8: Boolean = false) { private val input = DataInputStream(ByteArrayInputStream(payload)) private var pushed = -1 @@ -6306,7 +6311,10 @@ class MessagePackReader(payload: ByteArray) { code == 0xdb -> input.readInt() else -> throw IOException("expected string") } - return String(readPayload(length), StandardCharsets.UTF_8) + val bytes = readPayload(length) + if (!strictUtf8) return String(bytes, StandardCharsets.UTF_8) + return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString() } private fun readBinaryAfterCode(code: Int): ByteArray { diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt new file mode 100644 index 00000000..7bc4bdc8 --- /dev/null +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt @@ -0,0 +1,129 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +package org.gamecult.cultmesh + +// (low, high) pairs, flattened and sorted. +internal val LETTER_OR_NUMBER: IntArray = intArrayOf( + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, + 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, + 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, + 0x124A, 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, + 0x12B2, 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, + 0x1318, 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, + 0x1681, 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, + 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, + 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, + 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, + 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, + 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, + 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, + 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, + 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, + 0x2107, 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, + 0x212A, 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, + 0x24EA, 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, + 0x2D27, 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, + 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, + 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, + 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, + 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, + 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, + 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, + 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, + 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, + 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, + 0xAAE0, 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB30, 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, + 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, + 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, + 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, + 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, + 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, + 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, + 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, + 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, + 0x10767, 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, + 0x10835, 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, + 0x108AF, 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, + 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, + 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, + 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, + 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, + 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, + 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, + 0x11144, 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, + 0x111DA, 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, + 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, + 0x112F9, 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, + 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, + 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, + 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, + 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, + 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, + 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, + 0x119D0, 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, + 0x11A50, 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, + 0x11C08, 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, + 0x11D09, 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, + 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, + 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, + 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, + 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, + 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, + 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, + 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, + 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, + 0x1BC88, 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, + 0x1D454, 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, + 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, + 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, + 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, + 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, + 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, + 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, + 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, + 0x1ECAB, 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, + 0x1EE1F, 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, + 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, + 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, + 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, + 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, + 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, + 0x323AF, +) + +internal val DEFAULT_IGNORABLE: IntArray = intArrayOf( + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +) diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt index 4b38d942..9cf9c402 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceipt.kt @@ -74,7 +74,36 @@ fun decodeWriteAdmitted(message: CultNetMessage): CultNetWriteAdmitted { closedKeys(message, setOf("messageId", "operation", "schemaId", "recordKey")) val operation = nonEmpty(message, "operation") if (operation != "put" && operation != "delete") throw IOException("operation must be put or delete") - return CultNetWriteAdmitted(nonEmpty(message, "messageId"), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) + return CultNetWriteAdmitted(admittedMessageId(message), operation, nonEmpty(message, "schemaId"), nonEmpty(message, "recordKey")) +} + +// write-receipt.md: a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no Default_Ignorable code point anywhere, as of +// Unicode 16.0, compared raw (never trimmed). The tables are embedded (UnicodeMessageIdTables.kt, generated by +// scripts/gen-unicode-messageid-tables.py), not the JDK's, so the running JVM's Unicode version changes no result; the other +// four runtimes embed the same generated tables. +private fun inRanges(ranges: IntArray, point: Int): Boolean { + var low = 0 + var high = ranges.size / 2 - 1 + while (low <= high) { + val mid = (low + high) ushr 1 + when { + point < ranges[2 * mid] -> high = mid - 1 + point > ranges[2 * mid + 1] -> low = mid + 1 + else -> return true + } + } + return false +} + +private fun admittedMessageId(message: CultNetMessage): String { + val id = nonEmpty(message, "messageId") + var hasLetterOrNumber = false + id.codePoints().forEach { cp -> + if (inRanges(DEFAULT_IGNORABLE, cp)) throw IOException("messageId holds a Default_Ignorable code point") + if (inRanges(LETTER_OR_NUMBER, cp)) hasLetterOrNumber = true + } + if (!hasLetterOrNumber) throw IOException("messageId must contain a letter or number") + return id } /** Strict decode of error.v0 (no messageId, no limit_reached) or error.v1 (messageId required, string or nil, never empty). */ diff --git a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt index e527692d..b7215a12 100644 --- a/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt +++ b/packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/WriteReceiptVectors.kt @@ -48,6 +48,72 @@ fun replayWriteReceiptVectors(path: String): Boolean { } } + // admittedMessageId: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, compared + // raw. The frame is built with this runtime's own encoder; a case with `wireHex` (the lone surrogate, invalid UTF-8) + // supplies the whole frame. + for (case in list(obj(vectors["admittedMessageId"])["cases"])) { + val c = obj(case) + check("admittedMessageId/${c["name"]}") { + val id = list(c["codePoints"]).joinToString("") { String(Character.toChars((it as String).toInt(16))) }.repeat(((c["repeat"] as Long?) ?: 1L).toInt()) + val bytes = (c["wireHex"] as String?)?.let(::unhex) ?: MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray() + val decoded = try { decodeReceiptFrame(parseCultNetMessage(bytes)) } catch (_: IOException) { null } + when { + c["expect"] == "refuse" -> if (decoded == null) emptyList() else listOf("accepted an id that must be refused (${c["why"]})") + decoded == null -> listOf("refused an id that must be accepted (${c["why"]})") + decoded.toWireMap()["messageId"] != id -> listOf("id not echoed raw: ${decoded.toWireMap()["messageId"]}") + else -> emptyList() + } + } + } + + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every + // range, so deleting or narrowing any range fails here. + check("unicodeTableRanges") { + @Suppress("UNCHECKED_CAST") + val ranges = JsonReader(File(File(path).parentFile, "write-admitted-message-id-ranges.json").readText(Charsets.UTF_8)).readDocument() as Map + fun pairs(key: String) = list(ranges[key]).map { list(it).map { n -> (n as Long).toInt() } } + val letters = pairs("letterOrNumber") + val ignorable = pairs("defaultIgnorable") + val problems = mutableListOf() + if (letters.size <= 700 || ignorable.size != 17) problems += "ranges file holds ${letters.size} and ${ignorable.size} ranges" + fun accepted(id: String): Boolean = try { + decodeReceiptFrame(parseCultNetMessage(MessagePackWriter().value(linkedMapOf( + "schemaVersion" to WRITE_ADMITTED_VERSION, "messageId" to id, "operation" to "put", "schemaId" to "note.v0", "recordKey" to "note:1", + )).toByteArray())) + true + } catch (_: IOException) { false } + for ((low, high) in letters) for ((point, want) in listOf(low to true, high to true, low - 1 to false, high + 1 to false)) { + if (accepted(String(Character.toChars(point))) != want) problems += "U+${point.toString(16)} alone should be ${if (want) "accepted" else "refused"}" + } + for ((low, high) in ignorable) for ((point, want) in listOf(low to false, high to false, low - 1 to true, high + 1 to true)) { + if (accepted("a" + String(Character.toChars(point))) != want) problems += "a + U+${point.toString(16)} should be ${if (want) "accepted" else "refused"}" + } + problems + } + + // A document payload string with a lone surrogate (ED A0 80) is not a write_admitted id: the shared MessagePack reader keeps + // reading it. Strictness is scoped to the write_admitted frame, never to the reader. + check("documentPayloadStringWithLoneSurrogateStillDecodes") { + val text = MessagePackReader(unhex("A3EDA080")).readString() + if (text.isNotEmpty()) emptyList() else listOf("decoded to an empty string") + } + + // admittedMessageIdDuplicates: Kotlin decodes the frame to a map, so it cannot see a repeated key: it judges the last one. + for (case in list(obj(vectors["admittedMessageIdDuplicates"])["cases"])) { + val c = obj(case) + check("admittedMessageIdDuplicates/${c["name"]}") { + val decoded = try { decodeReceiptFrame(parseCultNetMessage(unhex(c["wireHex"] as String))) } catch (_: IOException) { null } + when { + c["lastKeyWins"] == "refuse" -> if (decoded == null) emptyList() else listOf("accepted a frame whose last messageId key is invalid") + decoded == null -> listOf("refused a frame whose last messageId key is valid") + decoded.toWireMap()["messageId"] != c["lastKeyWinsEcho"] -> listOf("echoed ${decoded.toWireMap()["messageId"]}, wanted ${c["lastKeyWinsEcho"]}") + else -> emptyList() + } + } + } + for (case in list(vectors["schemaInvalid"])) { val c = obj(case) check("schemaInvalid/${c["name"]}") { @@ -129,8 +195,9 @@ fun replayWriteReceiptVectors(path: String): Boolean { val loaded = mapOf( "wire" to list(vectors["wire"]).size, "schemaInvalid" to list(vectors["schemaInvalid"]).size, "contentHashCases" to list(obj(vectors["contentHash"])["cases"]).size, "scenarioCases" to list(obj(vectors["scenarios"])["cases"]).size, + "admittedMessageIdCases" to list(obj(vectors["admittedMessageId"])["cases"]).size, ) - val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/")) + val executed = mapOf("wire" to ran("wire/"), "schemaInvalid" to ran("schemaInvalid/"), "contentHashCases" to ran("contentHash/"), "scenarioCases" to ran("scenario/"), "admittedMessageIdCases" to ran("admittedMessageId/")) loaded.keys.filter { loaded[it] != (declared[it] as Long).toInt() || executed[it] != loaded[it] } .map { "$it: declared ${declared[it]}, loaded ${loaded[it]}, ran ${executed[it]}" } } diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index f2d89296..d5e26b7a 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -6,6 +6,8 @@ from typing import Any import msgpack +from .unicode_messageid_tables import DEFAULT_IGNORABLE, LETTER_OR_NUMBER + @dataclass(frozen=True) class CultNetMessage: @@ -69,6 +71,36 @@ def _is_name(value: Any) -> bool: return isinstance(value, str) and value != "" +# write-receipt.md: a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no Default_Ignorable code point anywhere, as of +# Unicode 16.0, compared raw (never stripped). The tables are embedded (unicode_messageid_tables.py, generated by +# scripts/gen-unicode-messageid-tables.py), not unicodedata's, so the running Python's Unicode version changes no result; +# the other four runtimes embed the same generated tables. +def _in_ranges(ranges: tuple[int, ...], point: int) -> bool: + """Whether [point] lies in the flattened, sorted (low, high) pairs.""" + low, high = 0, len(ranges) // 2 - 1 + while low <= high: + mid = (low + high) // 2 + if point < ranges[2 * mid]: + high = mid - 1 + elif point > ranges[2 * mid + 1]: + low = mid + 1 + else: + return True + return False + + +def _is_admitted_message_id(value: Any) -> bool: + if not isinstance(value, str): + return False + has_letter_or_number = False + for ch in value: + point = ord(ch) + if _in_ranges(DEFAULT_IGNORABLE, point): + return False + has_letter_or_number = has_letter_or_number or _in_ranges(LETTER_OR_NUMBER, point) + return has_letter_or_number + + def _validate_write_admitted_body(body: dict[str, Any]) -> None: """Refuse a cultnet.write_admitted.v0 body that is not exactly the five keys of its closed schema.""" expected = ("messageId", "operation", "schemaId", "recordKey") @@ -77,6 +109,8 @@ def _validate_write_admitted_body(body: dict[str, Any]) -> None: for key in ("messageId", "schemaId", "recordKey"): if not _is_name(body[key]): raise ValueError(f"cultnet.write_admitted.v0 {key} must be a non-empty string") + if not _is_admitted_message_id(body["messageId"]): + raise ValueError("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point") if body["operation"] not in ("put", "delete"): raise ValueError("cultnet.write_admitted.v0 operation must be put or delete") diff --git a/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py b/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py new file mode 100644 index 00000000..163e3c08 --- /dev/null +++ b/packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py @@ -0,0 +1,127 @@ +# generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +# (low, high) pairs, flattened and sorted. +LETTER_OR_NUMBER: tuple[int, ...] = ( + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, + 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, + 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, + 0x124A, 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, + 0x12B2, 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, + 0x1318, 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, + 0x1681, 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, + 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, + 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, + 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, + 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, + 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, + 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, + 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, + 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, + 0x2107, 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, + 0x212A, 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, + 0x24EA, 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, + 0x2D27, 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, + 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, + 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, + 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, + 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, + 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, + 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, + 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, + 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, + 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, + 0xAAE0, 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB30, 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, + 0xF900, 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, + 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, + 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, + 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, + 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, + 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, + 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, + 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, + 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, + 0x10767, 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, + 0x10835, 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, + 0x108AF, 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, + 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, + 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, + 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, + 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, + 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, + 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, + 0x11144, 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, + 0x111DA, 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, + 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, + 0x112F9, 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, + 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, + 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, + 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, + 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, + 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, + 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, + 0x119D0, 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, + 0x11A50, 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, + 0x11C08, 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, + 0x11D09, 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, + 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, + 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, + 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, + 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, + 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, + 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, + 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, + 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, + 0x1BC88, 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, + 0x1D454, 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, + 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, + 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, + 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, + 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, + 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, + 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, + 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, + 0x1ECAB, 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, + 0x1EE1F, 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, + 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, + 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, + 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, + 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, + 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, + 0x323AF, +) + +DEFAULT_IGNORABLE: tuple[int, ...] = ( + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +) diff --git a/packages/cultnet-py/tests/test_write_receipt.py b/packages/cultnet-py/tests/test_write_receipt.py index 98e74287..0ff1d017 100644 --- a/packages/cultnet-py/tests/test_write_receipt.py +++ b/packages/cultnet-py/tests/test_write_receipt.py @@ -57,7 +57,7 @@ class WriteReceiptVectorTests(unittest.TestCase): def test_vector_sections_are_the_ones_this_file_accounts_for(self) -> None: self.assertEqual( set(_VECTORS) - {"provenance", "specification"}, - {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts"}, + {"constants", "contentHash", "dedupe", "wire", "blankFields", "schemaInvalid", "scenarios", "counts", "admittedMessageId", "admittedMessageIdDuplicates"}, ) self.assertEqual(len(_VECTORS["contentHash"]["cases"]), 4) self.assertEqual(len(_VECTORS["wire"]), 7) @@ -68,6 +68,69 @@ class WriteReceiptVectorTests(unittest.TestCase): self.assertEqual(counts["schemaInvalid"], len(_VECTORS["schemaInvalid"])) self.assertEqual(counts["contentHashCases"], len(_VECTORS["contentHash"]["cases"])) self.assertEqual(counts["scenarioCases"], len(_VECTORS["scenarios"]["cases"])) + self.assertEqual(counts["admittedMessageIdCases"], len(_VECTORS["admittedMessageId"]["cases"])) + + def test_admitted_message_id_vectors_are_refused_or_accepted_raw(self) -> None: + # `admittedMessageId`: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, + # compared raw (never trimmed). A lone surrogate is not valid UTF-8, so its vector carries the whole frame. + cases = _VECTORS["admittedMessageId"]["cases"] + failures = [] + for case in cases: + raw = ( + bytes.fromhex(case["wireHex"]) + if "wireHex" in case + else _pack(_admitted(messageId="".join(chr(int(cp, 16)) for cp in case["codePoints"]) * case.get("repeat", 1))) + ) + try: + message = parse_message(raw) + except Exception: # noqa: BLE001 - any refusal counts + refused = True + else: + refused = False + wanted = "".join(chr(int(cp, 16)) for cp in case["codePoints"]) * case.get("repeat", 1) + if message.body["messageId"] != wanted: + failures.append(f"{case['name']}: id not echoed raw") + if refused != (case["expect"] == "refuse"): + failures.append(f"{case['name']}: expected {case['expect']}") + self.assertEqual(failures, []) + + def test_a_repeated_message_id_key_follows_the_last_key_wins_policy(self) -> None: + # Python decodes the frame to a dict, so it cannot see a duplicate: it judges the last key (admittedMessageIdDuplicates). + section = _VECTORS["admittedMessageIdDuplicates"] + self.assertEqual(_VECTORS["counts"]["admittedMessageIdDuplicateCases"], len(section["cases"])) + for case in section["cases"]: + raw = bytes.fromhex(case["wireHex"]) + if case["lastKeyWins"] == "refuse": + with self.assertRaises(ValueError, msg=case["name"]): + parse_message(raw) + else: + self.assertEqual(parse_message(raw).body["messageId"], case["lastKeyWinsEcho"], case["name"]) + + def test_every_range_of_the_unicode_tables_is_pinned_at_both_ends(self) -> None: + # write-admitted-message-id-ranges.json is the generated, independent copy: for each letter-or-number range its first and + # last point are accepted and the points either side refused; for each Default_Ignorable range "a" beside its first and + # last point is refused and beside the points either side accepted. Deleting or narrowing any embedded range fails here. + ranges = json.loads((_INTEROP / "write-admitted-message-id-ranges.json").read_text(encoding="utf-8")) + self.assertGreater(len(ranges["letterOrNumber"]), 700) + self.assertEqual(len(ranges["defaultIgnorable"]), 17) + + def accepted(text: str) -> bool: + try: + parse_message(_pack(_admitted(messageId=text))) + return True + except ValueError: + return False + + failures = [] + for low, high in ranges["letterOrNumber"]: + for point, want in ((low, True), (high, True), (low - 1, False), (high + 1, False)): + if accepted(chr(point)) != want: + failures.append(f"U+{point:04X} alone should be {'accepted' if want else 'refused'}") + for low, high in ranges["defaultIgnorable"]: + for point, want in ((low, False), (high, False), (low - 1, True), (high + 1, True)): + if accepted("a" + chr(point)) != want: + failures.append(f"a + U+{point:04X} should be {'accepted' if want else 'refused'}") + self.assertEqual(failures, []) def test_content_hash_matches_every_vector(self) -> None: for case in _VECTORS["contentHash"]["cases"]: diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index 0ea2aa53..bf907806 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -680,7 +680,7 @@ fn validate_message(message: &CultNetMessage) -> Result<()> { } } CultNetMessage::WriteAdmitted { message_id, operation, schema_id, record_key } => { - require_non_empty(message_id, "messageId")?; + require_admitted_message_id(message_id)?; if operation != "put" && operation != "delete" { return Err(anyhow!("operation must be put or delete")); } @@ -934,6 +934,41 @@ fn require_non_empty(value: &str, field: &str) -> Result<()> { Ok(()) } +// write-receipt.md: a write_admitted messageId is valid iff it holds a Unicode letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) and no +// Default_Ignorable_Code_Point, as of Unicode 16.0, compared raw (never trimmed). The tables are embedded +// (unicode_messageid_tables.rs, generated by scripts/gen-unicode-messageid-tables.py), not std's, so no platform's Unicode +// version changes a result; the other four runtimes embed the same generated tables. Invalid UTF-8 never reaches this check: +// the MessagePack str decode refuses it. +use crate::unicode_messageid_tables::{DEFAULT_IGNORABLE, LETTER_OR_NUMBER}; + +fn in_ranges(ranges: &[(u32, u32)], point: u32) -> bool { + ranges.binary_search_by(|&(low, high)| { + if point < low { + std::cmp::Ordering::Greater + } else if point > high { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} + +fn require_admitted_message_id(value: &str) -> Result<()> { + let mut has_letter_or_number = false; + for ch in value.chars() { + let point = ch as u32; + if in_ranges(DEFAULT_IGNORABLE, point) { + return Err(anyhow!("CultNet field messageId must not hold a Default_Ignorable code point")); + } + has_letter_or_number |= in_ranges(LETTER_OR_NUMBER, point); + } + if !has_letter_or_number { + return Err(anyhow!("CultNet field messageId must contain a letter or number")); + } + Ok(()) +} + fn require_exact(value: &str, expected: &str, field: &str) -> Result<()> { if value != expected { return Err(anyhow!("{field} must be {expected}")); diff --git a/packages/cultnet-rs/src/lib.rs b/packages/cultnet-rs/src/lib.rs index b35bd701..e008a2ba 100644 --- a/packages/cultnet-rs/src/lib.rs +++ b/packages/cultnet-rs/src/lib.rs @@ -3,6 +3,7 @@ mod contracts; mod idunn_deployment_brake; mod idunn_lifecycle_brake; mod media_fec; +mod unicode_messageid_tables; mod media_stream_contracts; mod media_stream_wire; mod provider_health_contracts; diff --git a/packages/cultnet-rs/src/unicode_messageid_tables.rs b/packages/cultnet-rs/src/unicode_messageid_tables.rs new file mode 100644 index 00000000..b5293f7a --- /dev/null +++ b/packages/cultnet-rs/src/unicode_messageid_tables.rs @@ -0,0 +1,147 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +// (low, high) pairs, sorted. +pub const LETTER_OR_NUMBER: &[(u32, u32)] = &[ + (0x30, 0x39), (0x41, 0x5A), (0x61, 0x7A), (0xAA, 0xAA), (0xB2, 0xB3), (0xB5, 0xB5), (0xB9, 0xBA), (0xBC, 0xBE), + (0xC0, 0xD6), (0xD8, 0xF6), (0xF8, 0x2C1), (0x2C6, 0x2D1), (0x2E0, 0x2E4), (0x2EC, 0x2EC), (0x2EE, 0x2EE), + (0x370, 0x374), (0x376, 0x377), (0x37A, 0x37D), (0x37F, 0x37F), (0x386, 0x386), (0x388, 0x38A), (0x38C, 0x38C), + (0x38E, 0x3A1), (0x3A3, 0x3F5), (0x3F7, 0x481), (0x48A, 0x52F), (0x531, 0x556), (0x559, 0x559), (0x560, 0x588), + (0x5D0, 0x5EA), (0x5EF, 0x5F2), (0x620, 0x64A), (0x660, 0x669), (0x66E, 0x66F), (0x671, 0x6D3), (0x6D5, 0x6D5), + (0x6E5, 0x6E6), (0x6EE, 0x6FC), (0x6FF, 0x6FF), (0x710, 0x710), (0x712, 0x72F), (0x74D, 0x7A5), (0x7B1, 0x7B1), + (0x7C0, 0x7EA), (0x7F4, 0x7F5), (0x7FA, 0x7FA), (0x800, 0x815), (0x81A, 0x81A), (0x824, 0x824), (0x828, 0x828), + (0x840, 0x858), (0x860, 0x86A), (0x870, 0x887), (0x889, 0x88E), (0x8A0, 0x8C9), (0x904, 0x939), (0x93D, 0x93D), + (0x950, 0x950), (0x958, 0x961), (0x966, 0x96F), (0x971, 0x980), (0x985, 0x98C), (0x98F, 0x990), (0x993, 0x9A8), + (0x9AA, 0x9B0), (0x9B2, 0x9B2), (0x9B6, 0x9B9), (0x9BD, 0x9BD), (0x9CE, 0x9CE), (0x9DC, 0x9DD), (0x9DF, 0x9E1), + (0x9E6, 0x9F1), (0x9F4, 0x9F9), (0x9FC, 0x9FC), (0xA05, 0xA0A), (0xA0F, 0xA10), (0xA13, 0xA28), (0xA2A, 0xA30), + (0xA32, 0xA33), (0xA35, 0xA36), (0xA38, 0xA39), (0xA59, 0xA5C), (0xA5E, 0xA5E), (0xA66, 0xA6F), (0xA72, 0xA74), + (0xA85, 0xA8D), (0xA8F, 0xA91), (0xA93, 0xAA8), (0xAAA, 0xAB0), (0xAB2, 0xAB3), (0xAB5, 0xAB9), (0xABD, 0xABD), + (0xAD0, 0xAD0), (0xAE0, 0xAE1), (0xAE6, 0xAEF), (0xAF9, 0xAF9), (0xB05, 0xB0C), (0xB0F, 0xB10), (0xB13, 0xB28), + (0xB2A, 0xB30), (0xB32, 0xB33), (0xB35, 0xB39), (0xB3D, 0xB3D), (0xB5C, 0xB5D), (0xB5F, 0xB61), (0xB66, 0xB6F), + (0xB71, 0xB77), (0xB83, 0xB83), (0xB85, 0xB8A), (0xB8E, 0xB90), (0xB92, 0xB95), (0xB99, 0xB9A), (0xB9C, 0xB9C), + (0xB9E, 0xB9F), (0xBA3, 0xBA4), (0xBA8, 0xBAA), (0xBAE, 0xBB9), (0xBD0, 0xBD0), (0xBE6, 0xBF2), (0xC05, 0xC0C), + (0xC0E, 0xC10), (0xC12, 0xC28), (0xC2A, 0xC39), (0xC3D, 0xC3D), (0xC58, 0xC5A), (0xC5D, 0xC5D), (0xC60, 0xC61), + (0xC66, 0xC6F), (0xC78, 0xC7E), (0xC80, 0xC80), (0xC85, 0xC8C), (0xC8E, 0xC90), (0xC92, 0xCA8), (0xCAA, 0xCB3), + (0xCB5, 0xCB9), (0xCBD, 0xCBD), (0xCDD, 0xCDE), (0xCE0, 0xCE1), (0xCE6, 0xCEF), (0xCF1, 0xCF2), (0xD04, 0xD0C), + (0xD0E, 0xD10), (0xD12, 0xD3A), (0xD3D, 0xD3D), (0xD4E, 0xD4E), (0xD54, 0xD56), (0xD58, 0xD61), (0xD66, 0xD78), + (0xD7A, 0xD7F), (0xD85, 0xD96), (0xD9A, 0xDB1), (0xDB3, 0xDBB), (0xDBD, 0xDBD), (0xDC0, 0xDC6), (0xDE6, 0xDEF), + (0xE01, 0xE30), (0xE32, 0xE33), (0xE40, 0xE46), (0xE50, 0xE59), (0xE81, 0xE82), (0xE84, 0xE84), (0xE86, 0xE8A), + (0xE8C, 0xEA3), (0xEA5, 0xEA5), (0xEA7, 0xEB0), (0xEB2, 0xEB3), (0xEBD, 0xEBD), (0xEC0, 0xEC4), (0xEC6, 0xEC6), + (0xED0, 0xED9), (0xEDC, 0xEDF), (0xF00, 0xF00), (0xF20, 0xF33), (0xF40, 0xF47), (0xF49, 0xF6C), (0xF88, 0xF8C), + (0x1000, 0x102A), (0x103F, 0x1049), (0x1050, 0x1055), (0x105A, 0x105D), (0x1061, 0x1061), (0x1065, 0x1066), + (0x106E, 0x1070), (0x1075, 0x1081), (0x108E, 0x108E), (0x1090, 0x1099), (0x10A0, 0x10C5), (0x10C7, 0x10C7), + (0x10CD, 0x10CD), (0x10D0, 0x10FA), (0x10FC, 0x1248), (0x124A, 0x124D), (0x1250, 0x1256), (0x1258, 0x1258), + (0x125A, 0x125D), (0x1260, 0x1288), (0x128A, 0x128D), (0x1290, 0x12B0), (0x12B2, 0x12B5), (0x12B8, 0x12BE), + (0x12C0, 0x12C0), (0x12C2, 0x12C5), (0x12C8, 0x12D6), (0x12D8, 0x1310), (0x1312, 0x1315), (0x1318, 0x135A), + (0x1369, 0x137C), (0x1380, 0x138F), (0x13A0, 0x13F5), (0x13F8, 0x13FD), (0x1401, 0x166C), (0x166F, 0x167F), + (0x1681, 0x169A), (0x16A0, 0x16EA), (0x16EE, 0x16F8), (0x1700, 0x1711), (0x171F, 0x1731), (0x1740, 0x1751), + (0x1760, 0x176C), (0x176E, 0x1770), (0x1780, 0x17B3), (0x17D7, 0x17D7), (0x17DC, 0x17DC), (0x17E0, 0x17E9), + (0x17F0, 0x17F9), (0x1810, 0x1819), (0x1820, 0x1878), (0x1880, 0x1884), (0x1887, 0x18A8), (0x18AA, 0x18AA), + (0x18B0, 0x18F5), (0x1900, 0x191E), (0x1946, 0x196D), (0x1970, 0x1974), (0x1980, 0x19AB), (0x19B0, 0x19C9), + (0x19D0, 0x19DA), (0x1A00, 0x1A16), (0x1A20, 0x1A54), (0x1A80, 0x1A89), (0x1A90, 0x1A99), (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), (0x1B45, 0x1B4C), (0x1B50, 0x1B59), (0x1B83, 0x1BA0), (0x1BAE, 0x1BE5), (0x1C00, 0x1C23), + (0x1C40, 0x1C49), (0x1C4D, 0x1C7D), (0x1C80, 0x1C8A), (0x1C90, 0x1CBA), (0x1CBD, 0x1CBF), (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), (0x1CF5, 0x1CF6), (0x1CFA, 0x1CFA), (0x1D00, 0x1DBF), (0x1E00, 0x1F15), (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), (0x1F48, 0x1F4D), (0x1F50, 0x1F57), (0x1F59, 0x1F59), (0x1F5B, 0x1F5B), (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), (0x1F80, 0x1FB4), (0x1FB6, 0x1FBC), (0x1FBE, 0x1FBE), (0x1FC2, 0x1FC4), (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), (0x1FD6, 0x1FDB), (0x1FE0, 0x1FEC), (0x1FF2, 0x1FF4), (0x1FF6, 0x1FFC), (0x2070, 0x2071), + (0x2074, 0x2079), (0x207F, 0x2089), (0x2090, 0x209C), (0x2102, 0x2102), (0x2107, 0x2107), (0x210A, 0x2113), + (0x2115, 0x2115), (0x2119, 0x211D), (0x2124, 0x2124), (0x2126, 0x2126), (0x2128, 0x2128), (0x212A, 0x212D), + (0x212F, 0x2139), (0x213C, 0x213F), (0x2145, 0x2149), (0x214E, 0x214E), (0x2150, 0x2189), (0x2460, 0x249B), + (0x24EA, 0x24FF), (0x2776, 0x2793), (0x2C00, 0x2CE4), (0x2CEB, 0x2CEE), (0x2CF2, 0x2CF3), (0x2CFD, 0x2CFD), + (0x2D00, 0x2D25), (0x2D27, 0x2D27), (0x2D2D, 0x2D2D), (0x2D30, 0x2D67), (0x2D6F, 0x2D6F), (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), (0x2DA8, 0x2DAE), (0x2DB0, 0x2DB6), (0x2DB8, 0x2DBE), (0x2DC0, 0x2DC6), (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), (0x2DD8, 0x2DDE), (0x2E2F, 0x2E2F), (0x3005, 0x3007), (0x3021, 0x3029), (0x3031, 0x3035), + (0x3038, 0x303C), (0x3041, 0x3096), (0x309D, 0x309F), (0x30A1, 0x30FA), (0x30FC, 0x30FF), (0x3105, 0x312F), + (0x3131, 0x318E), (0x3192, 0x3195), (0x31A0, 0x31BF), (0x31F0, 0x31FF), (0x3220, 0x3229), (0x3248, 0x324F), + (0x3251, 0x325F), (0x3280, 0x3289), (0x32B1, 0x32BF), (0x3400, 0x4DBF), (0x4E00, 0xA48C), (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), (0xA610, 0xA62B), (0xA640, 0xA66E), (0xA67F, 0xA69D), (0xA6A0, 0xA6EF), (0xA717, 0xA71F), + (0xA722, 0xA788), (0xA78B, 0xA7CD), (0xA7D0, 0xA7D1), (0xA7D3, 0xA7D3), (0xA7D5, 0xA7DC), (0xA7F2, 0xA801), + (0xA803, 0xA805), (0xA807, 0xA80A), (0xA80C, 0xA822), (0xA830, 0xA835), (0xA840, 0xA873), (0xA882, 0xA8B3), + (0xA8D0, 0xA8D9), (0xA8F2, 0xA8F7), (0xA8FB, 0xA8FB), (0xA8FD, 0xA8FE), (0xA900, 0xA925), (0xA930, 0xA946), + (0xA960, 0xA97C), (0xA984, 0xA9B2), (0xA9CF, 0xA9D9), (0xA9E0, 0xA9E4), (0xA9E6, 0xA9FE), (0xAA00, 0xAA28), + (0xAA40, 0xAA42), (0xAA44, 0xAA4B), (0xAA50, 0xAA59), (0xAA60, 0xAA76), (0xAA7A, 0xAA7A), (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), (0xAAB5, 0xAAB6), (0xAAB9, 0xAABD), (0xAAC0, 0xAAC0), (0xAAC2, 0xAAC2), (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), (0xAAF2, 0xAAF4), (0xAB01, 0xAB06), (0xAB09, 0xAB0E), (0xAB11, 0xAB16), (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), (0xAB30, 0xAB5A), (0xAB5C, 0xAB69), (0xAB70, 0xABE2), (0xABF0, 0xABF9), (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), (0xD7CB, 0xD7FB), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0xFB00, 0xFB06), (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), (0xFB1F, 0xFB28), (0xFB2A, 0xFB36), (0xFB38, 0xFB3C), (0xFB3E, 0xFB3E), (0xFB40, 0xFB41), + (0xFB43, 0xFB44), (0xFB46, 0xFBB1), (0xFBD3, 0xFD3D), (0xFD50, 0xFD8F), (0xFD92, 0xFDC7), (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), (0xFE76, 0xFEFC), (0xFF10, 0xFF19), (0xFF21, 0xFF3A), (0xFF41, 0xFF5A), (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), (0xFFCA, 0xFFCF), (0xFFD2, 0xFFD7), (0xFFDA, 0xFFDC), (0x10000, 0x1000B), (0x1000D, 0x10026), + (0x10028, 0x1003A), (0x1003C, 0x1003D), (0x1003F, 0x1004D), (0x10050, 0x1005D), (0x10080, 0x100FA), + (0x10107, 0x10133), (0x10140, 0x10178), (0x1018A, 0x1018B), (0x10280, 0x1029C), (0x102A0, 0x102D0), + (0x102E1, 0x102FB), (0x10300, 0x10323), (0x1032D, 0x1034A), (0x10350, 0x10375), (0x10380, 0x1039D), + (0x103A0, 0x103C3), (0x103C8, 0x103CF), (0x103D1, 0x103D5), (0x10400, 0x1049D), (0x104A0, 0x104A9), + (0x104B0, 0x104D3), (0x104D8, 0x104FB), (0x10500, 0x10527), (0x10530, 0x10563), (0x10570, 0x1057A), + (0x1057C, 0x1058A), (0x1058C, 0x10592), (0x10594, 0x10595), (0x10597, 0x105A1), (0x105A3, 0x105B1), + (0x105B3, 0x105B9), (0x105BB, 0x105BC), (0x105C0, 0x105F3), (0x10600, 0x10736), (0x10740, 0x10755), + (0x10760, 0x10767), (0x10780, 0x10785), (0x10787, 0x107B0), (0x107B2, 0x107BA), (0x10800, 0x10805), + (0x10808, 0x10808), (0x1080A, 0x10835), (0x10837, 0x10838), (0x1083C, 0x1083C), (0x1083F, 0x10855), + (0x10858, 0x10876), (0x10879, 0x1089E), (0x108A7, 0x108AF), (0x108E0, 0x108F2), (0x108F4, 0x108F5), + (0x108FB, 0x1091B), (0x10920, 0x10939), (0x10980, 0x109B7), (0x109BC, 0x109CF), (0x109D2, 0x10A00), + (0x10A10, 0x10A13), (0x10A15, 0x10A17), (0x10A19, 0x10A35), (0x10A40, 0x10A48), (0x10A60, 0x10A7E), + (0x10A80, 0x10A9F), (0x10AC0, 0x10AC7), (0x10AC9, 0x10AE4), (0x10AEB, 0x10AEF), (0x10B00, 0x10B35), + (0x10B40, 0x10B55), (0x10B58, 0x10B72), (0x10B78, 0x10B91), (0x10BA9, 0x10BAF), (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), (0x10CC0, 0x10CF2), (0x10CFA, 0x10D23), (0x10D30, 0x10D39), (0x10D40, 0x10D65), + (0x10D6F, 0x10D85), (0x10E60, 0x10E7E), (0x10E80, 0x10EA9), (0x10EB0, 0x10EB1), (0x10EC2, 0x10EC4), + (0x10F00, 0x10F27), (0x10F30, 0x10F45), (0x10F51, 0x10F54), (0x10F70, 0x10F81), (0x10FB0, 0x10FCB), + (0x10FE0, 0x10FF6), (0x11003, 0x11037), (0x11052, 0x1106F), (0x11071, 0x11072), (0x11075, 0x11075), + (0x11083, 0x110AF), (0x110D0, 0x110E8), (0x110F0, 0x110F9), (0x11103, 0x11126), (0x11136, 0x1113F), + (0x11144, 0x11144), (0x11147, 0x11147), (0x11150, 0x11172), (0x11176, 0x11176), (0x11183, 0x111B2), + (0x111C1, 0x111C4), (0x111D0, 0x111DA), (0x111DC, 0x111DC), (0x111E1, 0x111F4), (0x11200, 0x11211), + (0x11213, 0x1122B), (0x1123F, 0x11240), (0x11280, 0x11286), (0x11288, 0x11288), (0x1128A, 0x1128D), + (0x1128F, 0x1129D), (0x1129F, 0x112A8), (0x112B0, 0x112DE), (0x112F0, 0x112F9), (0x11305, 0x1130C), + (0x1130F, 0x11310), (0x11313, 0x11328), (0x1132A, 0x11330), (0x11332, 0x11333), (0x11335, 0x11339), + (0x1133D, 0x1133D), (0x11350, 0x11350), (0x1135D, 0x11361), (0x11380, 0x11389), (0x1138B, 0x1138B), + (0x1138E, 0x1138E), (0x11390, 0x113B5), (0x113B7, 0x113B7), (0x113D1, 0x113D1), (0x113D3, 0x113D3), + (0x11400, 0x11434), (0x11447, 0x1144A), (0x11450, 0x11459), (0x1145F, 0x11461), (0x11480, 0x114AF), + (0x114C4, 0x114C5), (0x114C7, 0x114C7), (0x114D0, 0x114D9), (0x11580, 0x115AE), (0x115D8, 0x115DB), + (0x11600, 0x1162F), (0x11644, 0x11644), (0x11650, 0x11659), (0x11680, 0x116AA), (0x116B8, 0x116B8), + (0x116C0, 0x116C9), (0x116D0, 0x116E3), (0x11700, 0x1171A), (0x11730, 0x1173B), (0x11740, 0x11746), + (0x11800, 0x1182B), (0x118A0, 0x118F2), (0x118FF, 0x11906), (0x11909, 0x11909), (0x1190C, 0x11913), + (0x11915, 0x11916), (0x11918, 0x1192F), (0x1193F, 0x1193F), (0x11941, 0x11941), (0x11950, 0x11959), + (0x119A0, 0x119A7), (0x119AA, 0x119D0), (0x119E1, 0x119E1), (0x119E3, 0x119E3), (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), (0x11A3A, 0x11A3A), (0x11A50, 0x11A50), (0x11A5C, 0x11A89), (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), (0x11BC0, 0x11BE0), (0x11BF0, 0x11BF9), (0x11C00, 0x11C08), (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), (0x11C50, 0x11C6C), (0x11C72, 0x11C8F), (0x11D00, 0x11D06), (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), (0x11D46, 0x11D46), (0x11D50, 0x11D59), (0x11D60, 0x11D65), (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), (0x11D98, 0x11D98), (0x11DA0, 0x11DA9), (0x11EE0, 0x11EF2), (0x11F02, 0x11F02), + (0x11F04, 0x11F10), (0x11F12, 0x11F33), (0x11F50, 0x11F59), (0x11FB0, 0x11FB0), (0x11FC0, 0x11FD4), + (0x12000, 0x12399), (0x12400, 0x1246E), (0x12480, 0x12543), (0x12F90, 0x12FF0), (0x13000, 0x1342F), + (0x13441, 0x13446), (0x13460, 0x143FA), (0x14400, 0x14646), (0x16100, 0x1611D), (0x16130, 0x16139), + (0x16800, 0x16A38), (0x16A40, 0x16A5E), (0x16A60, 0x16A69), (0x16A70, 0x16ABE), (0x16AC0, 0x16AC9), + (0x16AD0, 0x16AED), (0x16B00, 0x16B2F), (0x16B40, 0x16B43), (0x16B50, 0x16B59), (0x16B5B, 0x16B61), + (0x16B63, 0x16B77), (0x16B7D, 0x16B8F), (0x16D40, 0x16D6C), (0x16D70, 0x16D79), (0x16E40, 0x16E96), + (0x16F00, 0x16F4A), (0x16F50, 0x16F50), (0x16F93, 0x16F9F), (0x16FE0, 0x16FE1), (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), (0x18800, 0x18CD5), (0x18CFF, 0x18D08), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B122), (0x1B132, 0x1B132), (0x1B150, 0x1B152), (0x1B155, 0x1B155), + (0x1B164, 0x1B167), (0x1B170, 0x1B2FB), (0x1BC00, 0x1BC6A), (0x1BC70, 0x1BC7C), (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), (0x1CCF0, 0x1CCF9), (0x1D2C0, 0x1D2D3), (0x1D2E0, 0x1D2F3), (0x1D360, 0x1D378), + (0x1D400, 0x1D454), (0x1D456, 0x1D49C), (0x1D49E, 0x1D49F), (0x1D4A2, 0x1D4A2), (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), (0x1D4AE, 0x1D4B9), (0x1D4BB, 0x1D4BB), (0x1D4BD, 0x1D4C3), (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), (0x1D50D, 0x1D514), (0x1D516, 0x1D51C), (0x1D51E, 0x1D539), (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), (0x1D546, 0x1D546), (0x1D54A, 0x1D550), (0x1D552, 0x1D6A5), (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), (0x1D6DC, 0x1D6FA), (0x1D6FC, 0x1D714), (0x1D716, 0x1D734), (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), (0x1D770, 0x1D788), (0x1D78A, 0x1D7A8), (0x1D7AA, 0x1D7C2), (0x1D7C4, 0x1D7CB), + (0x1D7CE, 0x1D7FF), (0x1DF00, 0x1DF1E), (0x1DF25, 0x1DF2A), (0x1E030, 0x1E06D), (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), (0x1E140, 0x1E149), (0x1E14E, 0x1E14E), (0x1E290, 0x1E2AD), (0x1E2C0, 0x1E2EB), + (0x1E2F0, 0x1E2F9), (0x1E4D0, 0x1E4EB), (0x1E4F0, 0x1E4F9), (0x1E5D0, 0x1E5ED), (0x1E5F0, 0x1E5FA), + (0x1E7E0, 0x1E7E6), (0x1E7E8, 0x1E7EB), (0x1E7ED, 0x1E7EE), (0x1E7F0, 0x1E7FE), (0x1E800, 0x1E8C4), + (0x1E8C7, 0x1E8CF), (0x1E900, 0x1E943), (0x1E94B, 0x1E94B), (0x1E950, 0x1E959), (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), (0x1ECB1, 0x1ECB4), (0x1ED01, 0x1ED2D), (0x1ED2F, 0x1ED3D), (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), (0x1EE21, 0x1EE22), (0x1EE24, 0x1EE24), (0x1EE27, 0x1EE27), (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), (0x1EE39, 0x1EE39), (0x1EE3B, 0x1EE3B), (0x1EE42, 0x1EE42), (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), (0x1EE4B, 0x1EE4B), (0x1EE4D, 0x1EE4F), (0x1EE51, 0x1EE52), (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), (0x1EE59, 0x1EE59), (0x1EE5B, 0x1EE5B), (0x1EE5D, 0x1EE5D), (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), (0x1EE64, 0x1EE64), (0x1EE67, 0x1EE6A), (0x1EE6C, 0x1EE72), (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), (0x1EE7E, 0x1EE7E), (0x1EE80, 0x1EE89), (0x1EE8B, 0x1EE9B), (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), (0x1EEAB, 0x1EEBB), (0x1F100, 0x1F10C), (0x1FBF0, 0x1FBF9), (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), (0x30000, 0x3134A), (0x31350, 0x323AF), +]; + +pub const DEFAULT_IGNORABLE: &[(u32, u32)] = &[ + (0xAD, 0xAD), (0x34F, 0x34F), (0x61C, 0x61C), (0x115F, 0x1160), (0x17B4, 0x17B5), (0x180B, 0x180F), + (0x200B, 0x200F), (0x202A, 0x202E), (0x2060, 0x206F), (0x3164, 0x3164), (0xFE00, 0xFE0F), (0xFEFF, 0xFEFF), + (0xFFA0, 0xFFA0), (0xFFF0, 0xFFF8), (0x1BCA0, 0x1BCA3), (0x1D173, 0x1D17A), (0xE0000, 0xE0FFF), +]; diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs index 3d233a61..3dd02a76 100644 --- a/packages/cultnet-rs/tests/write_receipt.rs +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -153,3 +153,103 @@ fn error_v0_refuses_the_v1_only_limit_reached_code() { as_v0[at + 15] = b'0'; assert!(decode_cultnet_message_from_slice(&as_v0, CultNetWireContract::CultNetSchemaV0).is_err()); } + +/// Vectors `admittedMessageId`: a write_admitted messageId needs a letter or number and no Default_Ignorable code +/// point, compared raw (never trimmed). The frame is the spec's `write_admitted_put` bytes with the id swapped in; a +/// case with `wireHex` (the lone surrogate, invalid UTF-8) supplies the whole frame. +#[test] +fn write_admitted_message_id_vectors_are_refused_or_accepted_raw() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + let vectors: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = vectors["admittedMessageId"]["cases"].as_array().unwrap(); + assert_eq!(vectors["counts"]["admittedMessageIdCases"].as_u64().unwrap() as usize, cases.len()); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let mut failures = Vec::new(); + for case in cases { + let name = case["name"].as_str().unwrap(); + let id: String = case["codePoints"] + .as_array() + .unwrap() + .iter() + .filter_map(|cp| char::from_u32(u32::from_str_radix(cp.as_str().unwrap(), 16).unwrap())) + .collect::() + .repeat(case["repeat"].as_u64().unwrap_or(1) as usize); + let frame = match case["wireHex"].as_str() { + Some(hex) => unhex(hex), + None => { + let len = id.len(); + let header: Vec = if len < 32 { + vec![0xA0 | len as u8] + } else if len < 256 { + vec![0xD9, len as u8] + } else { + vec![0xDA, (len >> 8) as u8, len as u8] + }; + [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat() + } + }; + let decoded = decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0); + match (case["expect"].as_str().unwrap(), decoded) { + ("refuse", Ok(_)) => failures.push(format!("{name}: expected refuse")), + ("accept", Err(error)) => failures.push(format!("{name}: expected accept, got {error}")), + ("accept", Ok(CultNetMessage::WriteAdmitted { message_id, .. })) if message_id != id => { + failures.push(format!("{name}: id not echoed raw")) + } + _ => {} + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} + +/// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +#[test] +fn every_range_of_the_unicode_tables_is_pinned_at_both_ends() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-admitted-message-id-ranges.json"); + let ranges: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let letters = ranges["letterOrNumber"].as_array().unwrap(); + let ignorable = ranges["defaultIgnorable"].as_array().unwrap(); + assert!(letters.len() > 700); + assert_eq!(ignorable.len(), 17); + let template = unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"); + let old_id_at = template.windows(5).position(|w| w == b"put-1").unwrap() - 1; + let accepted = |id: &str| { + let header = vec![0xA0 | id.len() as u8]; + let frame = [&template[..old_id_at], &header[..], id.as_bytes(), &template[old_id_at + 6..]].concat(); + decode_cultnet_message_from_slice(&frame, CultNetWireContract::CultNetSchemaV0).is_ok() + }; + let point = |value: i64| char::from_u32(value as u32).unwrap().to_string(); + let mut failures = Vec::new(); + for range in letters { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, true), (high, true), (low - 1, false), (high + 1, false)] { + if accepted(&point(at)) != want { + failures.push(format!("U+{at:X} alone should be {}", if want { "accepted" } else { "refused" })); + } + } + } + for range in ignorable { + let (low, high) = (range[0].as_i64().unwrap(), range[1].as_i64().unwrap()); + for (at, want) in [(low, false), (high, false), (low - 1, true), (high + 1, true)] { + if accepted(&format!("a{}", point(at))) != want { + failures.push(format!("a + U+{at:X} should be {}", if want { "accepted" } else { "refused" })); + } + } + } + assert!(failures.is_empty(), "{} failing:\n{}", failures.len(), failures.join("\n")); +} + +/// Vectors `admittedMessageIdDuplicates`: a repeated messageId key is refused whatever the values. This pins the outcome, not the +/// cause (today serde's `duplicate field messageId` in the typed decode of write_admitted). +#[test] +fn a_repeated_message_id_key_is_refused_whatever_the_values() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../contracts/cultnet/interop/write-receipt-vectors.json"); + let vectors: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let cases = vectors["admittedMessageIdDuplicates"]["cases"].as_array().unwrap(); + assert_eq!(vectors["counts"]["admittedMessageIdDuplicateCases"].as_u64().unwrap() as usize, cases.len()); + for case in cases { + assert_eq!(case["seesDuplicate"], "refuse"); + let decoded = decode_cultnet_message_from_slice(&unhex(case["wireHex"].as_str().unwrap()), CultNetWireContract::CultNetSchemaV0); + assert!(decoded.is_err(), "{}: a repeated messageId key must be refused", case["name"]); + } +} diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 3cc87dcd..8dd37ced 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -10,6 +10,7 @@ import verifySchema from "../contracts/cultnet/cultnet.verify.schema.json"; import loginSuccessSchema from "../contracts/cultnet/cultnet.login-success.schema.json"; import errorSchema from "../contracts/cultnet/cultnet.error.schema.json"; import errorV1Schema from "../contracts/cultnet/interop/cultnet.error.v1.schema.json"; +import { DEFAULT_IGNORABLE, LETTER_OR_NUMBER } from "./unicode-messageid-tables"; import writeAdmittedSchema from "../contracts/cultnet/interop/cultnet.write-admitted.schema.json"; import sampleChangeNameSchema from "../contracts/cultnet/cultnet.sample-change-name.schema.json"; import sampleChatSchema from "../contracts/cultnet/cultnet.sample-chat.schema.json"; @@ -950,6 +951,10 @@ export function parseCultNetMessage(input: unknown, wireContract: CultNetWireCon return input as CultNetDatabaseChangeRawMessage; } + if (schemaVersion === "cultnet.write_admitted.v0" && !isValidAdmittedMessageId((input as { messageId?: unknown }).messageId)) { + throw new Error("Validation failed for cultnet.write_admitted.v0: /messageId: must contain a letter or number and no Default_Ignorable code point"); + } + if (!validator(input)) { const keys = Object.keys(input as Record).join(","); throw new Error(`${renderValidationErrors(schemaVersion, validator)}; keys=${keys}`); @@ -1460,6 +1465,45 @@ function validateRawDocumentRecord( requireOptionalStringArray(candidate.tags, `${instancePath}/tags`); } +/** + * A write_admitted messageId is valid iff it holds a Unicode letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) and no + * Default_Ignorable_Code_Point, as of Unicode 16.0, compared raw (write-receipt.md; the id is echoed byte for byte, so nothing + * is trimmed). The tables are embedded (unicode-messageid-tables.ts, generated by scripts/gen-unicode-messageid-tables.py), not + * the regex engine's, so the running Node's Unicode version changes no result; the other four runtimes embed the same tables. + */ +function isValidAdmittedMessageId(value: unknown): boolean { + if (typeof value !== "string") { + return false; + } + let hasLetterOrNumber = false; + for (const ch of value) { + const point = ch.codePointAt(0) as number; + // The decoded string must be well-formed Unicode: @msgpack/msgpack turns invalid UTF-8 (ED A0 80) into a lone surrogate. + if ((point >= 0xd800 && point <= 0xdfff) || inRanges(DEFAULT_IGNORABLE, point)) { + return false; + } + hasLetterOrNumber = hasLetterOrNumber || inRanges(LETTER_OR_NUMBER, point); + } + return hasLetterOrNumber; +} + +/** Whether [point] lies in the flattened, sorted [low, high] pairs. */ +function inRanges(ranges: readonly number[], point: number): boolean { + let low = 0; + let high = ranges.length / 2 - 1; + while (low <= high) { + const mid = (low + high) >> 1; + if (point < ranges[2 * mid]) { + high = mid - 1; + } else if (point > ranges[2 * mid + 1]) { + low = mid + 1; + } else { + return true; + } + } + return false; +} + /** Blank is empty or only Unicode White_Space (write-receipt.md, rule 5), not `trim()`: JS trims U+FEFF and keeps U+0085. */ function isBlank(value: string): boolean { return /^\p{White_Space}*$/u.test(value); diff --git a/packages/cultnet-ts/src/unicode-messageid-tables.ts b/packages/cultnet-ts/src/unicode-messageid-tables.ts new file mode 100644 index 00000000..3374a011 --- /dev/null +++ b/packages/cultnet-ts/src/unicode-messageid-tables.ts @@ -0,0 +1,126 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +// [low, high] pairs, sorted. +export const LETTER_OR_NUMBER: readonly number[] = [ + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, 0xD6, 0xD8, + 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, 0x376, 0x377, 0x37A, + 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3F5, 0x3F7, 0x481, 0x48A, + 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, 0x620, 0x64A, 0x660, 0x669, 0x66E, + 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, + 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, + 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, + 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, + 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, + 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, + 0xA5C, 0xA5E, 0xA5E, 0xA66, 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, + 0xAB3, 0xAB5, 0xAB9, 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, + 0xB10, 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, + 0xB6F, 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, 0xC12, + 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, 0xC7E, 0xC80, + 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, 0xCDD, 0xCDE, 0xCE0, + 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, 0xD3D, 0xD4E, 0xD4E, 0xD54, + 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, + 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, + 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, + 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, 0x103F, + 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, + 0x108E, 0x1090, 0x1099, 0x10A0, 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, 0x124A, + 0x124D, 0x1250, 0x1256, 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, 0x12B2, + 0x12B5, 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, 0x1318, + 0x135A, 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, 0x1681, + 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, 0x176E, + 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, 0x1819, 0x1820, + 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1946, 0x196D, 0x1970, + 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, 0x1A54, 0x1A80, 0x1A89, 0x1A90, + 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1C00, + 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, 0x1CEE, + 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, + 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, 0x1FB6, + 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, 0x1FF2, + 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, 0x2107, + 0x2107, 0x210A, 0x2113, 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, + 0x212D, 0x212F, 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, 0x24EA, + 0x24FF, 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, 0x2D27, + 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, 0x2DB0, + 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, 0x2E2F, 0x3005, + 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, 0x30A1, 0x30FA, 0x30FC, + 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, 0x31FF, 0x3220, 0x3229, 0x3248, + 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, 0xA500, + 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, 0xA78B, + 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, + 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, 0xA8FD, + 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, 0xA9E6, + 0xA9FE, 0xAA00, 0xAA28, 0xAA40, 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7E, + 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, 0xAAE0, + 0xAAEA, 0xAAF2, 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, 0xAB30, + 0xAB5A, 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0xF900, + 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, 0xFB38, + 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, 0xFD92, + 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, 0xFF41, 0xFF5A, 0xFF66, + 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, 0x1000B, 0x1000D, 0x10026, + 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, 0x10080, 0x100FA, 0x10107, 0x10133, + 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x102E1, 0x102FB, 0x10300, 0x10323, + 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, 0x103D5, + 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, 0x10563, + 0x10570, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, 0x105B1, + 0x105B3, 0x105B9, 0x105BB, 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, + 0x10780, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, + 0x10837, 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, 0x109CF, + 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, 0x10A60, 0x10A7E, + 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, 0x10B35, 0x10B40, 0x10B55, + 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, + 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, 0x10D85, 0x10E60, 0x10E7E, 0x10E80, 0x10EA9, + 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, 0x10F81, + 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, 0x11075, + 0x11083, 0x110AF, 0x110D0, 0x110E8, 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, 0x11144, + 0x11147, 0x11147, 0x11150, 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, 0x111DA, + 0x111DC, 0x111DC, 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, 0x11286, + 0x11288, 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, 0x112F9, + 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, 0x11339, + 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, 0x1138E, + 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, 0x11434, 0x11447, 0x1144A, + 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, 0x114C7, 0x114C7, 0x114D0, 0x114D9, + 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, 0x11644, 0x11650, 0x11659, 0x11680, 0x116AA, + 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, 0x11746, + 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, 0x11916, + 0x11918, 0x1192F, 0x1193F, 0x1193F, 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D0, + 0x119E1, 0x119E1, 0x119E3, 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, 0x11A50, + 0x11A5C, 0x11A89, 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, + 0x11C0A, 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, 0x11D09, + 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, 0x11D89, + 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, 0x11F12, 0x11F33, + 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, 0x1246E, 0x12480, 0x12543, + 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, 0x14400, 0x14646, 0x16100, 0x1611D, + 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, 0x16AC9, + 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, + 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, 0x16F50, + 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, 0x18D08, + 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, 0x1B152, + 0x1B155, 0x1B155, 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, 0x1BC88, + 0x1BC90, 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, 0x1D4B9, + 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, 0x1D51C, + 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, 0x1D550, 0x1D552, 0x1D6A5, + 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, 0x1D716, 0x1D734, 0x1D736, 0x1D74E, + 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, 0x1D7FF, + 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, 0x1E149, + 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, 0x1E4F9, + 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, 0x1E7FE, + 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, 0x1ECAB, + 0x1ECAD, 0x1ECAF, 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, + 0x1EE21, 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, 0x1EE39, + 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, 0x1EE4F, + 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, 0x1EE5D, 0x1EE5D, + 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, 0x1EE77, + 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, 0x1EEA5, 0x1EEA9, + 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, 0x2B81D, + 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, 0x323AF, +]; + +export const DEFAULT_IGNORABLE: readonly number[] = [ + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, + 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, 0xFFF8, 0x1BCA0, + 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, +]; diff --git a/packages/cultnet-ts/test/write-receipt.test.ts b/packages/cultnet-ts/test/write-receipt.test.ts index 6acb79cf..23ef478e 100644 --- a/packages/cultnet-ts/test/write-receipt.test.ts +++ b/packages/cultnet-ts/test/write-receipt.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { decode } from "@msgpack/msgpack"; import { CULTNET_ERROR_CODES, CULTNET_ERROR_V1_CODES, @@ -280,3 +281,59 @@ test("a blank messageId, record key or tag is exactly Unicode White_Space, not t parseCultNetMessage(put({ tags: [named] })); } }); + +// `admittedMessageId` vectors: a write_admitted messageId needs a letter or number and no Default_Ignorable code point, +// compared raw (never trimmed). The D800 vector is the frame's bytes (wireHex), decoded by the msgpack library like the others. +test("write_admitted messageId: every admittedMessageId vector is refused or accepted raw", () => { + const vectorsPath = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + const vectors = JSON.parse(readFileSync(vectorsPath, "utf8")); + assert.equal(vectors.counts.admittedMessageIdCases, vectors.admittedMessageId.cases.length); + const failures: string[] = []; + for (const vector of vectors.admittedMessageId.cases as { name: string; codePoints: string[]; repeat?: number; wireHex?: string; expect: "refuse" | "accept" }[]) { + const id = vector.codePoints.map((cp) => String.fromCodePoint(parseInt(cp, 16))).join("").repeat(vector.repeat ?? 1); + let decodedId: string | undefined; + try { + const frame = vector.wireHex ? decode(Uint8Array.from(Buffer.from(vector.wireHex, "hex"))) : admitted(id); + decodedId = (parseCultNetMessage(frame) as CultNetWriteAdmittedMessage).messageId; + } catch { + decodedId = undefined; + } + if ((decodedId === undefined) !== (vector.expect === "refuse")) failures.push(`${vector.name}: expected ${vector.expect}`); + else if (decodedId !== undefined && decodedId !== id) failures.push(`${vector.name}: id not echoed raw`); + } + assert.deepEqual(failures, []); +}); + +// The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. +test("write_admitted messageId: every range of the Unicode 16.0 tables is pinned at both ends", () => { + const ranges = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"), "utf8")); + assert.ok(ranges.letterOrNumber.length > 700); + assert.equal(ranges.defaultIgnorable.length, 17); + const accepted = (id: string) => { try { parseCultNetMessage(admitted(id)); return true; } catch { return false; } }; + const failures: string[] = []; + for (const [low, high] of ranges.letterOrNumber as [number, number][]) { + for (const [point, want] of [[low, true], [high, true], [low - 1, false], [high + 1, false]] as [number, boolean][]) { + if (accepted(String.fromCodePoint(point)) !== want) failures.push(`U+${point.toString(16)} alone should be ${want ? "accepted" : "refused"}`); + } + } + for (const [low, high] of ranges.defaultIgnorable as [number, number][]) { + for (const [point, want] of [[low, false], [high, false], [low - 1, true], [high + 1, true]] as [number, boolean][]) { + if (accepted("a" + String.fromCodePoint(point)) !== want) failures.push(`a + U+${point.toString(16)} should be ${want ? "accepted" : "refused"}`); + } + } + assert.deepEqual(failures, []); +}); + +// TypeScript decodes the frame to an object, so it cannot see a duplicate key: it judges the last one (admittedMessageIdDuplicates). +// This pins @msgpack/msgpack's last-key behaviour plus the id check on the decoded object, not a first-key or duplicate path in +// cultnet-ts source: there is none, so a refuse-on-first mutation in src would not be killed. +test("write_admitted messageId: a repeated key follows the last-key-wins policy", () => { + const vectors = JSON.parse(readFileSync(join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "write-receipt-vectors.json"), "utf8")); + const section = vectors.admittedMessageIdDuplicates; + assert.equal(vectors.counts.admittedMessageIdDuplicateCases, section.cases.length); + for (const vector of section.cases as { name: string; wireHex: string; lastKeyWins: "accept" | "refuse"; lastKeyWinsEcho?: string }[]) { + const frame = decode(Uint8Array.from(Buffer.from(vector.wireHex, "hex"))); + if (vector.lastKeyWins === "refuse") assert.throws(() => parseCultNetMessage(frame), /./s, vector.name); + else assert.equal((parseCultNetMessage(frame) as CultNetWriteAdmittedMessage).messageId, vector.lastKeyWinsEcho, vector.name); + } +}); diff --git a/scripts/gen-unicode-messageid-tables.py b/scripts/gen-unicode-messageid-tables.py new file mode 100644 index 00000000..4c119930 --- /dev/null +++ b/scripts/gen-unicode-messageid-tables.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Generate the Unicode 16.0 tables of the write_admitted messageId rule for the five CultNet runtimes. + +The rule (contracts/cultnet/interop/write-receipt.md): a messageId is valid iff it holds a letter (Lu Ll Lt Lm Lo) or a number +(Nd Nl No) and no Default_Ignorable_Code_Point, all as of Unicode 16.0. The tables are embedded in each runtime so that no +platform's own Unicode version can change a result. Bumping the version is one coordinated change: run this script on the new +UCD files, commit the five regenerated files and the vectors that pin the new version, together. + + gen-unicode-messageid-tables.py DerivedGeneralCategory.txt DerivedCoreProperties.txt [--check] + +The inputs are https://www.unicode.org/Public/16.0.0/ucd/extracted/DerivedGeneralCategory.txt and +https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt. With --check nothing is written; the exit status is 1 if +any generated file differs from what these inputs produce (a stale or hand-edited table). Standard library only. +""" +import hashlib +import json +import re +import sys +from pathlib import Path + +VERSION = "16.0.0" +ROOT = Path(__file__).resolve().parent.parent +LETTER_OR_NUMBER_CATEGORIES = {"Lu", "Ll", "Lt", "Lm", "Lo", "Nd", "Nl", "No"} + +TARGETS = { + "ts": ROOT / "packages/cultnet-ts/src/unicode-messageid-tables.ts", + "py": ROOT / "packages/cultnet-py/src/cultnet_py/unicode_messageid_tables.py", + "kt": ROOT / "packages/cultmesh-kotlin/src/main/kotlin/org/gamecult/cultmesh/UnicodeMessageIdTables.kt", + "cs": ROOT / "src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs", + "rs": ROOT / "packages/cultnet-rs/src/unicode_messageid_tables.rs", +} +# The independent copy the runtimes' tests probe (first, last, and the code point either side of every range), so that deleting +# or narrowing any range of any embedded table fails a test. +RANGES_JSON = ROOT / "contracts/cultnet/interop/write-admitted-message-id-ranges.json" + + +def parse(path, wanted): + """Ranges (low, high) of the lines `LOW[..HIGH] ; VALUE`, VALUE accepted by [wanted], merged and sorted.""" + points = [] + for line in Path(path).read_text(encoding="utf-8").splitlines(): + m = re.match(r"([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*(\S+)", line) + if m and wanted(m.group(3)): + points.append((int(m.group(1), 16), int(m.group(2) or m.group(1), 16))) + points.sort() + merged = [] + for low, high in points: + if merged and low <= merged[-1][1] + 1: + merged[-1] = (merged[-1][0], max(merged[-1][1], high)) + else: + merged.append((low, high)) + return merged + + +def flat(ranges): + return [value for pair in ranges for value in pair] + + +def wrapped(items, indent, render, width=118): + lines, current = [], indent + for item in items: + text = render(item) + ", " + if len(current) + len(text) > width: + lines.append(current.rstrip()) + current = indent + current += text + lines.append(current.rstrip()) + return "\n".join(lines) + + +def emit(target, letters, ignorable, header): + num = lambda v: f"0x{v:X}" + if target == "ts": + body = lambda name, r: f"export const {name}: readonly number[] = [\n{wrapped(flat(r), ' ', num)}\n];\n" + return f"// {header}\n\n// [low, high] pairs, sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "py": + body = lambda name, r: f"{name}: tuple[int, ...] = (\n{wrapped(flat(r), ' ', num)}\n)\n" + return f"# {header}\n\n# (low, high) pairs, flattened and sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "kt": + body = lambda name, r: f"internal val {name}: IntArray = intArrayOf(\n{wrapped(flat(r), ' ', num)}\n)\n" + return f"// {header}\n\npackage org.gamecult.cultmesh\n\n// (low, high) pairs, flattened and sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + if target == "cs": + body = lambda name, r: f" public static readonly int[] {name} =\n {{\n{wrapped(flat(r), ' ', num)}\n }};\n" + return f"// {header}\n\nnamespace GameCult.Networking\n{{\n internal static class CultNetUnicodeMessageIdTables\n {{\n // (low, high) pairs, flattened and sorted.\n{body('LetterOrNumber', letters)}\n{body('DefaultIgnorable', ignorable)} }}\n}}\n" + body = lambda name, r: f"pub const {name}: &[(u32, u32)] = &[\n{wrapped(r, ' ', lambda p: f'({num(p[0])}, {num(p[1])})')}\n];\n" + return f"// {header}\n\n// (low, high) pairs, sorted.\n{body('LETTER_OR_NUMBER', letters)}\n{body('DEFAULT_IGNORABLE', ignorable)}" + + +def main(): + args = [a for a in sys.argv[1:] if a != "--check"] + check = "--check" in sys.argv[1:] + if len(args) != 2: + sys.exit(__doc__) + gc, dcp = args + for path, marker in ((gc, f"DerivedGeneralCategory-{VERSION}"), (dcp, f"DerivedCoreProperties-{VERSION}")): + if marker not in Path(path).read_text(encoding="utf-8").splitlines()[0]: + sys.exit(f"{path} is not {marker}.txt") + letters = parse(gc, lambda v: v in LETTER_OR_NUMBER_CATEGORIES) + ignorable = parse(dcp, lambda v: v == "Default_Ignorable_Code_Point") + digest = hashlib.sha256(b"".join(Path(p).read_bytes() for p in (gc, dcp))).hexdigest()[:16] + header = f"generated, Unicode {VERSION}, by scripts/gen-unicode-messageid-tables.py (inputs sha256 {digest}); do not edit" + stale = [] + texts = {path: emit(target, letters, ignorable, header) for target, path in TARGETS.items()} + texts[RANGES_JSON] = json.dumps( + {"generated": header, "unicodeVersion": VERSION, "letterOrNumber": [list(r) for r in letters], "defaultIgnorable": [list(r) for r in ignorable]}, + separators=(",", ":"), + ) + "\n" + for path, text in texts.items(): + if check: + if not path.exists() or path.read_text(encoding="utf-8") != text: + stale.append(str(path.relative_to(ROOT))) + else: + path.write_text(text, encoding="utf-8") + if stale: + sys.exit("stale or hand-edited: " + ", ".join(stale)) + print(f"{'checked' if check else 'wrote'} {len(texts)} files: {len(letters)} letter-or-number ranges, {len(ignorable)} default-ignorable ranges") + + +if __name__ == "__main__": + main() diff --git a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs index cdcca912..1e7bdd08 100644 --- a/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs +++ b/src/GameCult.Networking/CultNetSchemaMessageSerialization.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using MessagePack; namespace GameCult.Networking @@ -29,6 +30,7 @@ namespace GameCult.Networking public static T Deserialize(byte[] payload) where T : ICultNetSchemaMessage { if (payload == null) throw new ArgumentNullException(nameof(payload)); + if (typeof(T) == typeof(CultNetWriteAdmittedMessage)) CultNetAdmittedMessageId.Require(payload); return MessagePackSerializer.Deserialize(payload, Options); } @@ -98,4 +100,67 @@ namespace GameCult.Networking [Key("schemaVersion")] public string SchemaVersion { get; set; } = string.Empty; } } + + /// + /// The write_admitted messageId rule (write-receipt.md): a letter (Lu Ll Lt Lm Lo) or number (Nd Nl No) somewhere, no + /// Default_Ignorable code point anywhere, as of Unicode 16.0, wire bytes that are valid UTF-8, compared raw (never trimmed). + /// The tables are embedded (CultNetUnicodeMessageIdTables.cs, generated by scripts/gen-unicode-messageid-tables.py), not + /// .NET's, so the running runtime's Unicode version changes no result; the other four runtimes embed the same tables. + /// + internal static class CultNetAdmittedMessageId + { + private static readonly System.Text.UTF8Encoding StrictUtf8 = new(false, true); + + /// Throws unless the frame's one top-level messageId key is a valid write_admitted id (a repeated key is malformed); other keys are left to the serializer. + public static void Require(byte[] payload) + { + var reader = new MessagePackReader(payload); + var entries = reader.ReadMapHeader(); + var seen = false; + for (var index = 0; index < entries; index++) + { + var key = reader.NextMessagePackType == MessagePackType.String ? reader.ReadString() : null; + if (key == "messageId" && reader.NextMessagePackType == MessagePackType.String) + { + if (seen) throw new MessagePackSerializationException("cultnet.write_admitted.v0 carries messageId more than once."); + seen = true; + string id; + try { id = StrictUtf8.GetString(reader.ReadStringSequence()!.Value.ToArray()); } + catch (System.Text.DecoderFallbackException) { throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId is not valid UTF-8."); } + if (!IsValid(id)) throw new MessagePackSerializationException("cultnet.write_admitted.v0 messageId must contain a letter or number and no Default_Ignorable code point."); + continue; + } + + if (key == null) reader.Skip(); + reader.Skip(); + } + } + + public static bool IsValid(string id) + { + var hasLetterOrNumber = false; + for (var at = 0; at < id.Length; at += char.IsSurrogatePair(id, at) ? 2 : 1) + { + var point = char.IsSurrogatePair(id, at) ? char.ConvertToUtf32(id, at) : id[at]; + if (InRanges(CultNetUnicodeMessageIdTables.DefaultIgnorable, point)) return false; + hasLetterOrNumber |= InRanges(CultNetUnicodeMessageIdTables.LetterOrNumber, point); + } + + return hasLetterOrNumber; + } + + private static bool InRanges(int[] ranges, int point) + { + int low = 0, high = ranges.Length / 2 - 1; + while (low <= high) + { + var mid = (low + high) >> 1; + if (point < ranges[2 * mid]) high = mid - 1; + else if (point > ranges[2 * mid + 1]) low = mid + 1; + else return true; + } + + return false; + } + } } diff --git a/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs b/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs new file mode 100644 index 00000000..45cedbc5 --- /dev/null +++ b/src/GameCult.Networking/CultNetUnicodeMessageIdTables.cs @@ -0,0 +1,144 @@ +// generated, Unicode 16.0.0, by scripts/gen-unicode-messageid-tables.py (inputs sha256 24a07ef9fabf71e3); do not edit + +namespace GameCult.Networking +{ + internal static class CultNetUnicodeMessageIdTables + { + // (low, high) pairs, flattened and sorted. + public static readonly int[] LetterOrNumber = + { + 0x30, 0x39, 0x41, 0x5A, 0x61, 0x7A, 0xAA, 0xAA, 0xB2, 0xB3, 0xB5, 0xB5, 0xB9, 0xBA, 0xBC, 0xBE, 0xC0, + 0xD6, 0xD8, 0xF6, 0xF8, 0x2C1, 0x2C6, 0x2D1, 0x2E0, 0x2E4, 0x2EC, 0x2EC, 0x2EE, 0x2EE, 0x370, 0x374, + 0x376, 0x377, 0x37A, 0x37D, 0x37F, 0x37F, 0x386, 0x386, 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, + 0x3F5, 0x3F7, 0x481, 0x48A, 0x52F, 0x531, 0x556, 0x559, 0x559, 0x560, 0x588, 0x5D0, 0x5EA, 0x5EF, 0x5F2, + 0x620, 0x64A, 0x660, 0x669, 0x66E, 0x66F, 0x671, 0x6D3, 0x6D5, 0x6D5, 0x6E5, 0x6E6, 0x6EE, 0x6FC, 0x6FF, + 0x6FF, 0x710, 0x710, 0x712, 0x72F, 0x74D, 0x7A5, 0x7B1, 0x7B1, 0x7C0, 0x7EA, 0x7F4, 0x7F5, 0x7FA, 0x7FA, + 0x800, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, 0x828, 0x840, 0x858, 0x860, 0x86A, 0x870, 0x887, 0x889, + 0x88E, 0x8A0, 0x8C9, 0x904, 0x939, 0x93D, 0x93D, 0x950, 0x950, 0x958, 0x961, 0x966, 0x96F, 0x971, 0x980, + 0x985, 0x98C, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, 0x9B2, 0x9B2, 0x9B6, 0x9B9, 0x9BD, 0x9BD, 0x9CE, + 0x9CE, 0x9DC, 0x9DD, 0x9DF, 0x9E1, 0x9E6, 0x9F1, 0x9F4, 0x9F9, 0x9FC, 0x9FC, 0xA05, 0xA0A, 0xA0F, 0xA10, + 0xA13, 0xA28, 0xA2A, 0xA30, 0xA32, 0xA33, 0xA35, 0xA36, 0xA38, 0xA39, 0xA59, 0xA5C, 0xA5E, 0xA5E, 0xA66, + 0xA6F, 0xA72, 0xA74, 0xA85, 0xA8D, 0xA8F, 0xA91, 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, 0xAB3, 0xAB5, 0xAB9, + 0xABD, 0xABD, 0xAD0, 0xAD0, 0xAE0, 0xAE1, 0xAE6, 0xAEF, 0xAF9, 0xAF9, 0xB05, 0xB0C, 0xB0F, 0xB10, 0xB13, + 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB3D, 0xB3D, 0xB5C, 0xB5D, 0xB5F, 0xB61, 0xB66, 0xB6F, + 0xB71, 0xB77, 0xB83, 0xB83, 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, 0xB9C, 0xB9C, 0xB9E, + 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, 0xBAE, 0xBB9, 0xBD0, 0xBD0, 0xBE6, 0xBF2, 0xC05, 0xC0C, 0xC0E, 0xC10, + 0xC12, 0xC28, 0xC2A, 0xC39, 0xC3D, 0xC3D, 0xC58, 0xC5A, 0xC5D, 0xC5D, 0xC60, 0xC61, 0xC66, 0xC6F, 0xC78, + 0xC7E, 0xC80, 0xC80, 0xC85, 0xC8C, 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB3, 0xCB5, 0xCB9, 0xCBD, 0xCBD, + 0xCDD, 0xCDE, 0xCE0, 0xCE1, 0xCE6, 0xCEF, 0xCF1, 0xCF2, 0xD04, 0xD0C, 0xD0E, 0xD10, 0xD12, 0xD3A, 0xD3D, + 0xD3D, 0xD4E, 0xD4E, 0xD54, 0xD56, 0xD58, 0xD61, 0xD66, 0xD78, 0xD7A, 0xD7F, 0xD85, 0xD96, 0xD9A, 0xDB1, + 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, 0xDC6, 0xDE6, 0xDEF, 0xE01, 0xE30, 0xE32, 0xE33, 0xE40, 0xE46, 0xE50, + 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE86, 0xE8A, 0xE8C, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEB0, 0xEB2, 0xEB3, + 0xEBD, 0xEBD, 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xED0, 0xED9, 0xEDC, 0xEDF, 0xF00, 0xF00, 0xF20, 0xF33, 0xF40, + 0xF47, 0xF49, 0xF6C, 0xF88, 0xF8C, 0x1000, 0x102A, 0x103F, 0x1049, 0x1050, 0x1055, 0x105A, 0x105D, + 0x1061, 0x1061, 0x1065, 0x1066, 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, 0x108E, 0x1090, 0x1099, 0x10A0, + 0x10C5, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10FA, 0x10FC, 0x1248, 0x124A, 0x124D, 0x1250, 0x1256, + 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1288, 0x128A, 0x128D, 0x1290, 0x12B0, 0x12B2, 0x12B5, 0x12B8, + 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, 0x12D8, 0x1310, 0x1312, 0x1315, 0x1318, 0x135A, + 0x1369, 0x137C, 0x1380, 0x138F, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1401, 0x166C, 0x166F, 0x167F, 0x1681, + 0x169A, 0x16A0, 0x16EA, 0x16EE, 0x16F8, 0x1700, 0x1711, 0x171F, 0x1731, 0x1740, 0x1751, 0x1760, 0x176C, + 0x176E, 0x1770, 0x1780, 0x17B3, 0x17D7, 0x17D7, 0x17DC, 0x17DC, 0x17E0, 0x17E9, 0x17F0, 0x17F9, 0x1810, + 0x1819, 0x1820, 0x1878, 0x1880, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x18AA, 0x18B0, 0x18F5, 0x1900, 0x191E, + 0x1946, 0x196D, 0x1970, 0x1974, 0x1980, 0x19AB, 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x1A00, 0x1A16, 0x1A20, + 0x1A54, 0x1A80, 0x1A89, 0x1A90, 0x1A99, 0x1AA7, 0x1AA7, 0x1B05, 0x1B33, 0x1B45, 0x1B4C, 0x1B50, 0x1B59, + 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1C00, 0x1C23, 0x1C40, 0x1C49, 0x1C4D, 0x1C7D, 0x1C80, 0x1C8A, 0x1C90, + 0x1CBA, 0x1CBD, 0x1CBF, 0x1CE9, 0x1CEC, 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF6, 0x1CFA, 0x1CFA, 0x1D00, 0x1DBF, + 0x1E00, 0x1F15, 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, 0x1F4D, 0x1F50, 0x1F57, 0x1F59, 0x1F59, 0x1F5B, + 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F7D, 0x1F80, 0x1FB4, 0x1FB6, 0x1FBC, 0x1FBE, 0x1FBE, 0x1FC2, 0x1FC4, + 0x1FC6, 0x1FCC, 0x1FD0, 0x1FD3, 0x1FD6, 0x1FDB, 0x1FE0, 0x1FEC, 0x1FF2, 0x1FF4, 0x1FF6, 0x1FFC, 0x2070, + 0x2071, 0x2074, 0x2079, 0x207F, 0x2089, 0x2090, 0x209C, 0x2102, 0x2102, 0x2107, 0x2107, 0x210A, 0x2113, + 0x2115, 0x2115, 0x2119, 0x211D, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, 0x212D, 0x212F, + 0x2139, 0x213C, 0x213F, 0x2145, 0x2149, 0x214E, 0x214E, 0x2150, 0x2189, 0x2460, 0x249B, 0x24EA, 0x24FF, + 0x2776, 0x2793, 0x2C00, 0x2CE4, 0x2CEB, 0x2CEE, 0x2CF2, 0x2CF3, 0x2CFD, 0x2CFD, 0x2D00, 0x2D25, 0x2D27, + 0x2D27, 0x2D2D, 0x2D2D, 0x2D30, 0x2D67, 0x2D6F, 0x2D6F, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, 0x2DA8, 0x2DAE, + 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x2E2F, + 0x2E2F, 0x3005, 0x3007, 0x3021, 0x3029, 0x3031, 0x3035, 0x3038, 0x303C, 0x3041, 0x3096, 0x309D, 0x309F, + 0x30A1, 0x30FA, 0x30FC, 0x30FF, 0x3105, 0x312F, 0x3131, 0x318E, 0x3192, 0x3195, 0x31A0, 0x31BF, 0x31F0, + 0x31FF, 0x3220, 0x3229, 0x3248, 0x324F, 0x3251, 0x325F, 0x3280, 0x3289, 0x32B1, 0x32BF, 0x3400, 0x4DBF, + 0x4E00, 0xA48C, 0xA4D0, 0xA4FD, 0xA500, 0xA60C, 0xA610, 0xA62B, 0xA640, 0xA66E, 0xA67F, 0xA69D, 0xA6A0, + 0xA6EF, 0xA717, 0xA71F, 0xA722, 0xA788, 0xA78B, 0xA7CD, 0xA7D0, 0xA7D1, 0xA7D3, 0xA7D3, 0xA7D5, 0xA7DC, + 0xA7F2, 0xA801, 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, 0xA822, 0xA830, 0xA835, 0xA840, 0xA873, 0xA882, + 0xA8B3, 0xA8D0, 0xA8D9, 0xA8F2, 0xA8F7, 0xA8FB, 0xA8FB, 0xA8FD, 0xA8FE, 0xA900, 0xA925, 0xA930, 0xA946, + 0xA960, 0xA97C, 0xA984, 0xA9B2, 0xA9CF, 0xA9D9, 0xA9E0, 0xA9E4, 0xA9E6, 0xA9FE, 0xAA00, 0xAA28, 0xAA40, + 0xAA42, 0xAA44, 0xAA4B, 0xAA50, 0xAA59, 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7E, 0xAAAF, 0xAAB1, 0xAAB1, + 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAC2, 0xAADB, 0xAADD, 0xAAE0, 0xAAEA, 0xAAF2, + 0xAAF4, 0xAB01, 0xAB06, 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, 0xAB30, 0xAB5A, + 0xAB5C, 0xAB69, 0xAB70, 0xABE2, 0xABF0, 0xABF9, 0xAC00, 0xD7A3, 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0xF900, + 0xFA6D, 0xFA70, 0xFAD9, 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB28, 0xFB2A, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, + 0xFD8F, 0xFD92, 0xFDC7, 0xFDF0, 0xFDFB, 0xFE70, 0xFE74, 0xFE76, 0xFEFC, 0xFF10, 0xFF19, 0xFF21, 0xFF3A, + 0xFF41, 0xFF5A, 0xFF66, 0xFFBE, 0xFFC2, 0xFFC7, 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0x10000, + 0x1000B, 0x1000D, 0x10026, 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, + 0x10080, 0x100FA, 0x10107, 0x10133, 0x10140, 0x10178, 0x1018A, 0x1018B, 0x10280, 0x1029C, 0x102A0, + 0x102D0, 0x102E1, 0x102FB, 0x10300, 0x10323, 0x1032D, 0x1034A, 0x10350, 0x10375, 0x10380, 0x1039D, + 0x103A0, 0x103C3, 0x103C8, 0x103CF, 0x103D1, 0x103D5, 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x104B0, + 0x104D3, 0x104D8, 0x104FB, 0x10500, 0x10527, 0x10530, 0x10563, 0x10570, 0x1057A, 0x1057C, 0x1058A, + 0x1058C, 0x10592, 0x10594, 0x10595, 0x10597, 0x105A1, 0x105A3, 0x105B1, 0x105B3, 0x105B9, 0x105BB, + 0x105BC, 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, 0x10780, 0x10785, + 0x10787, 0x107B0, 0x107B2, 0x107BA, 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, 0x10837, + 0x10838, 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10858, 0x10876, 0x10879, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x10920, 0x10939, 0x10980, 0x109B7, 0x109BC, + 0x109CF, 0x109D2, 0x10A00, 0x10A10, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x10A40, 0x10A48, + 0x10A60, 0x10A7E, 0x10A80, 0x10A9F, 0x10AC0, 0x10AC7, 0x10AC9, 0x10AE4, 0x10AEB, 0x10AEF, 0x10B00, + 0x10B35, 0x10B40, 0x10B55, 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10BA9, 0x10BAF, 0x10C00, 0x10C48, + 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, 0x10CFA, 0x10D23, 0x10D30, 0x10D39, 0x10D40, 0x10D65, 0x10D6F, + 0x10D85, 0x10E60, 0x10E7E, 0x10E80, 0x10EA9, 0x10EB0, 0x10EB1, 0x10EC2, 0x10EC4, 0x10F00, 0x10F27, + 0x10F30, 0x10F45, 0x10F51, 0x10F54, 0x10F70, 0x10F81, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11003, + 0x11037, 0x11052, 0x1106F, 0x11071, 0x11072, 0x11075, 0x11075, 0x11083, 0x110AF, 0x110D0, 0x110E8, + 0x110F0, 0x110F9, 0x11103, 0x11126, 0x11136, 0x1113F, 0x11144, 0x11144, 0x11147, 0x11147, 0x11150, + 0x11172, 0x11176, 0x11176, 0x11183, 0x111B2, 0x111C1, 0x111C4, 0x111D0, 0x111DA, 0x111DC, 0x111DC, + 0x111E1, 0x111F4, 0x11200, 0x11211, 0x11213, 0x1122B, 0x1123F, 0x11240, 0x11280, 0x11286, 0x11288, + 0x11288, 0x1128A, 0x1128D, 0x1128F, 0x1129D, 0x1129F, 0x112A8, 0x112B0, 0x112DE, 0x112F0, 0x112F9, + 0x11305, 0x1130C, 0x1130F, 0x11310, 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, + 0x11339, 0x1133D, 0x1133D, 0x11350, 0x11350, 0x1135D, 0x11361, 0x11380, 0x11389, 0x1138B, 0x1138B, + 0x1138E, 0x1138E, 0x11390, 0x113B5, 0x113B7, 0x113B7, 0x113D1, 0x113D1, 0x113D3, 0x113D3, 0x11400, + 0x11434, 0x11447, 0x1144A, 0x11450, 0x11459, 0x1145F, 0x11461, 0x11480, 0x114AF, 0x114C4, 0x114C5, + 0x114C7, 0x114C7, 0x114D0, 0x114D9, 0x11580, 0x115AE, 0x115D8, 0x115DB, 0x11600, 0x1162F, 0x11644, + 0x11644, 0x11650, 0x11659, 0x11680, 0x116AA, 0x116B8, 0x116B8, 0x116C0, 0x116C9, 0x116D0, 0x116E3, + 0x11700, 0x1171A, 0x11730, 0x1173B, 0x11740, 0x11746, 0x11800, 0x1182B, 0x118A0, 0x118F2, 0x118FF, + 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, 0x11915, 0x11916, 0x11918, 0x1192F, 0x1193F, 0x1193F, + 0x11941, 0x11941, 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D0, 0x119E1, 0x119E1, 0x119E3, + 0x119E3, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, 0x11A50, 0x11A50, 0x11A5C, 0x11A89, + 0x11A9D, 0x11A9D, 0x11AB0, 0x11AF8, 0x11BC0, 0x11BE0, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, 0x11C0A, + 0x11C2E, 0x11C40, 0x11C40, 0x11C50, 0x11C6C, 0x11C72, 0x11C8F, 0x11D00, 0x11D06, 0x11D08, 0x11D09, + 0x11D0B, 0x11D30, 0x11D46, 0x11D46, 0x11D50, 0x11D59, 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, + 0x11D89, 0x11D98, 0x11D98, 0x11DA0, 0x11DA9, 0x11EE0, 0x11EF2, 0x11F02, 0x11F02, 0x11F04, 0x11F10, + 0x11F12, 0x11F33, 0x11F50, 0x11F59, 0x11FB0, 0x11FB0, 0x11FC0, 0x11FD4, 0x12000, 0x12399, 0x12400, + 0x1246E, 0x12480, 0x12543, 0x12F90, 0x12FF0, 0x13000, 0x1342F, 0x13441, 0x13446, 0x13460, 0x143FA, + 0x14400, 0x14646, 0x16100, 0x1611D, 0x16130, 0x16139, 0x16800, 0x16A38, 0x16A40, 0x16A5E, 0x16A60, + 0x16A69, 0x16A70, 0x16ABE, 0x16AC0, 0x16AC9, 0x16AD0, 0x16AED, 0x16B00, 0x16B2F, 0x16B40, 0x16B43, + 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, 0x16B7D, 0x16B8F, 0x16D40, 0x16D6C, 0x16D70, + 0x16D79, 0x16E40, 0x16E96, 0x16F00, 0x16F4A, 0x16F50, 0x16F50, 0x16F93, 0x16F9F, 0x16FE0, 0x16FE1, + 0x16FE3, 0x16FE3, 0x17000, 0x187F7, 0x18800, 0x18CD5, 0x18CFF, 0x18D08, 0x1AFF0, 0x1AFF3, 0x1AFF5, + 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, 0x1B132, 0x1B132, 0x1B150, 0x1B152, 0x1B155, 0x1B155, + 0x1B164, 0x1B167, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, 0x1BC80, 0x1BC88, 0x1BC90, + 0x1BC99, 0x1CCF0, 0x1CCF9, 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D360, 0x1D378, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, 0x1D4A9, 0x1D4AC, 0x1D4AE, + 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, + 0x1D516, 0x1D51C, 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, 0x1D54A, + 0x1D550, 0x1D552, 0x1D6A5, 0x1D6A8, 0x1D6C0, 0x1D6C2, 0x1D6DA, 0x1D6DC, 0x1D6FA, 0x1D6FC, 0x1D714, + 0x1D716, 0x1D734, 0x1D736, 0x1D74E, 0x1D750, 0x1D76E, 0x1D770, 0x1D788, 0x1D78A, 0x1D7A8, 0x1D7AA, + 0x1D7C2, 0x1D7C4, 0x1D7CB, 0x1D7CE, 0x1D7FF, 0x1DF00, 0x1DF1E, 0x1DF25, 0x1DF2A, 0x1E030, 0x1E06D, + 0x1E100, 0x1E12C, 0x1E137, 0x1E13D, 0x1E140, 0x1E149, 0x1E14E, 0x1E14E, 0x1E290, 0x1E2AD, 0x1E2C0, + 0x1E2EB, 0x1E2F0, 0x1E2F9, 0x1E4D0, 0x1E4EB, 0x1E4F0, 0x1E4F9, 0x1E5D0, 0x1E5ED, 0x1E5F0, 0x1E5FA, + 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, 0x1E7F0, 0x1E7FE, 0x1E800, 0x1E8C4, 0x1E8C7, + 0x1E8CF, 0x1E900, 0x1E943, 0x1E94B, 0x1E94B, 0x1E950, 0x1E959, 0x1EC71, 0x1ECAB, 0x1ECAD, 0x1ECAF, + 0x1ECB1, 0x1ECB4, 0x1ED01, 0x1ED2D, 0x1ED2F, 0x1ED3D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, 0x1EE21, + 0x1EE22, 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, 0x1EE39, 0x1EE39, + 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, + 0x1EE4F, 0x1EE51, 0x1EE52, 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, + 0x1EE5D, 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, 0x1EE67, 0x1EE6A, 0x1EE6C, + 0x1EE72, 0x1EE74, 0x1EE77, 0x1EE79, 0x1EE7C, 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, + 0x1EEA1, 0x1EEA3, 0x1EEA5, 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10C, 0x1FBF0, 0x1FBF9, 0x20000, + 0x2A6DF, 0x2A700, 0x2B739, 0x2B740, 0x2B81D, 0x2B820, 0x2CEA1, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, + 0x2F800, 0x2FA1D, 0x30000, 0x3134A, 0x31350, 0x323AF, + }; + + public static readonly int[] DefaultIgnorable = + { + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, + 0x202A, 0x202E, 0x2060, 0x206F, 0x3164, 0x3164, 0xFE00, 0xFE0F, 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0xFFF0, + 0xFFF8, 0x1BCA0, 0x1BCA3, 0x1D173, 0x1D17A, 0xE0000, 0xE0FFF, + }; + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs new file mode 100644 index 00000000..6c65734c --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetWriteAdmittedMessageIdTests.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // contracts/cultnet/interop/write-receipt-vectors.json `admittedMessageId`: a write_admitted messageId needs a letter or + // number and no Default_Ignorable code point, compared raw (never trimmed). The frame is built with this runtime's own + // encoder; the case with `wireHex` (a lone surrogate is invalid UTF-8) supplies the whole frame. + public sealed class CultNetWriteAdmittedMessageIdTests + { + private static JsonElement Vectors() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var path = Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-receipt-vectors.json"); + return JsonDocument.Parse(File.ReadAllText(path)).RootElement.Clone(); + } + + public static IEnumerable Cases() => + Vectors().GetProperty("admittedMessageId").GetProperty("cases").EnumerateArray().Select(c => c.GetProperty("name").GetString()!); + + [Test] + public void TheFileDeclaresHowManyCasesThisRunnerMustRun() => + Assert.That(Cases().Count(), Is.EqualTo(Vectors().GetProperty("counts").GetProperty("admittedMessageIdCases").GetInt32())); + + [TestCaseSource(nameof(Cases))] + public void AWriteAdmittedMessageIdIsRefusedOrAcceptedRaw(string name) + { + var vector = Vectors().GetProperty("admittedMessageId").GetProperty("cases").EnumerateArray() + .Single(c => c.GetProperty("name").GetString() == name); + var id = string.Concat(Enumerable.Repeat( + string.Concat(vector.GetProperty("codePoints").EnumerateArray() + .Select(cp => Convert.ToInt32(cp.GetString(), 16)).Where(point => point < 0xD800 || point > 0xDFFF).Select(char.ConvertFromUtf32)), + vector.TryGetProperty("repeat", out var repeat) ? repeat.GetInt32() : 1)); + var bytes = vector.TryGetProperty("wireHex", out var wire) + ? Convert.FromHexString(wire.GetString()!) + : CultNetSchemaMessageSerialization.Serialize(new CultNetWriteAdmittedMessage + { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" }); + + CultNetWriteAdmittedMessage? decoded = null; + try { decoded = (CultNetWriteAdmittedMessage)CultNetSchemaMessageSerialization.Deserialize(bytes); } + catch (Exception) { /* a refusal */ } + + if (vector.GetProperty("expect").GetString() == "refuse") + Assert.That(decoded, Is.Null, vector.GetProperty("why").GetString()); + else + { + Assert.That(decoded, Is.Not.Null, vector.GetProperty("why").GetString()); + Assert.That(decoded!.MessageId, Is.EqualTo(id)); + } + } + + // The generated ranges file is the independent copy of the embedded tables: first, last and the points either side of every range. + [Test] + public void EveryRangeOfTheUnicodeTablesIsPinnedAtBothEnds() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var ranges = JsonDocument.Parse(File.ReadAllText(Path.Combine(directory!.FullName, "contracts", "cultnet", "interop", "write-admitted-message-id-ranges.json"))).RootElement; + var letters = ranges.GetProperty("letterOrNumber").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + var ignorable = ranges.GetProperty("defaultIgnorable").EnumerateArray().Select(r => (r[0].GetInt32(), r[1].GetInt32())).ToArray(); + Assert.That(letters.Length, Is.GreaterThan(700)); + Assert.That(ignorable.Length, Is.EqualTo(17)); + static bool Accepted(string id) + { + try + { + CultNetSchemaMessageSerialization.Deserialize(CultNetSchemaMessageSerialization.Serialize( + new CultNetWriteAdmittedMessage { MessageId = id, Operation = "put", SchemaId = "note.v0", RecordKey = "note:1" })); + return true; + } + catch (Exception) { return false; } + } + + var failures = new List(); + foreach (var (low, high) in letters) + foreach (var (point, want) in new[] { (low, true), (high, true), (low - 1, false), (high + 1, false) }) + if (Accepted(char.ConvertFromUtf32(point)) != want) failures.Add($"U+{point:X} alone should be {(want ? "accepted" : "refused")}"); + foreach (var (low, high) in ignorable) + foreach (var (point, want) in new[] { (low, false), (high, false), (low - 1, true), (high + 1, true) }) + if (Accepted("a" + char.ConvertFromUtf32(point)) != want) failures.Add($"a + U+{point:X} should be {(want ? "accepted" : "refused")}"); + Assert.That(failures, Is.Empty); + } + + // admittedMessageIdDuplicates: C# reads the raw map, so it sees a repeated messageId key and refuses it whatever the values. + [Test] + public void ARepeatedMessageIdKeyIsRefusedWhateverTheValues() + { + var cases = Vectors().GetProperty("admittedMessageIdDuplicates").GetProperty("cases").EnumerateArray().ToArray(); + Assert.That(cases.Length, Is.EqualTo(Vectors().GetProperty("counts").GetProperty("admittedMessageIdDuplicateCases").GetInt32())); + foreach (var vector in cases) + { + Assert.That(vector.GetProperty("seesDuplicate").GetString(), Is.EqualTo("refuse")); + Assert.That(() => CultNetSchemaMessageSerialization.Deserialize(Convert.FromHexString(vector.GetProperty("wireHex").GetString()!)), + Throws.Exception, vector.GetProperty("name").GetString()); + } + } + } +}