Merge write-receipt-text-blank-rules (3b1b7697) into libby/write-receipt-kt-interop

contracts/cultnet/interop/write-receipt-vectors.json

@@ -111,8 +111,11 @@
     }
   ],
   "blankFields": {
-    "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or only whitespace. Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.",
+    "semantics": "A put whose document 'names nothing' is refused like a frame that does not read (error.v1 message_invalid, details.field \"message\", the write's id, the fixed text of error_v1_empty_schema_id_is_unreadable_and_names_the_write) and never applied, on every connection, receipt-capable or not. Each case starts from this valid document: schemaId note.v0, recordKey note:1, storedAt 2026-11-08T00:00:00Z, payload 81A16101 (hex), no source ids, no tags; 'set' replaces or adds the named fields. A blank string is empty or made only of Unicode White_Space code points (the list in whiteSpaceCodePoints, the PropList property, the same in every runtime; U+200B, U+FEFF, U+180E and U+001C to U+001F are not White_Space, though some platforms' whitespace tests accept some of them). Fields not listed are valid. 'unset' means the key is absent or nil, which is not blank: those cases are admitted.",
     "validPutPayloadHex": "81A16101",
+    "whiteSpaceCodePoints": ["0009","000A","000B","000C","000D","0020","0085","00A0","1680","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","200A","2028","2029","202F","205F","3000"],
+    "notWhiteSpaceCodePoints": ["200B","200C","200D","2060","FEFF","180E","001C","001D","001E","001F"],
+    "codePointCases": "For every code point in whiteSpaceCodePoints, each of schemaId, recordKey, storedAt, sourceRuntimeId, sourceAgentId, sourceRole and one tags entry, set alone to a string of just that code point (and to that code point repeated three times), is refused; for every code point in notWhiteSpaceCodePoints the same string is not blank and the put is admitted.",
     "cases": [
       { "name": "valid_document_is_admitted", "set": {}, "outcome": "admitted" },
       { "name": "empty_schema_id", "set": { "schemaId": "" }, "outcome": "refused" },
@@ -126,6 +129,21 @@
       { "name": "blank_source_agent_id", "set": { "sourceAgentId": "  " }, "outcome": "refused" },
       { "name": "blank_source_role", "set": { "sourceRole": "" }, "outcome": "refused" },
       { "name": "blank_tag", "set": { "tags": ["a", " "] }, "outcome": "refused" },
+      { "name": "nbsp_schema_id", "set": { "schemaId": "\u00a0" }, "outcome": "refused" },
+      { "name": "em_space_record_key", "set": { "recordKey": "\u2003\u2003" }, "outcome": "refused" },
+      { "name": "next_line_stored_at", "set": { "storedAt": "\u0085" }, "outcome": "refused" },
+      { "name": "line_separator_source_runtime_id", "set": { "sourceRuntimeId": "\u2028" }, "outcome": "refused" },
+      { "name": "ideographic_space_source_agent_id", "set": { "sourceAgentId": "\u3000" }, "outcome": "refused" },
+      { "name": "nbsp_source_role", "set": { "sourceRole": " \u00a0 " }, "outcome": "refused" },
+      { "name": "em_space_tag", "set": { "tags": ["a", "\u2003"] }, "outcome": "refused" },
+      { "name": "next_line_tag", "set": { "tags": ["\u0085"] }, "outcome": "refused" },
+      { "name": "zero_width_space_schema_id_is_a_name", "set": { "schemaId": "\u200b" }, "outcome": "admitted" },
+      { "name": "zero_width_space_record_key_is_a_name", "set": { "recordKey": "\u200b" }, "outcome": "admitted" },
+      { "name": "zero_width_space_tag_is_a_name", "set": { "tags": ["\u200b"] }, "outcome": "admitted" },
+      { "name": "bom_stored_at_is_a_name", "set": { "storedAt": "\ufeff" }, "outcome": "admitted" },
+      { "name": "mongolian_vowel_separator_source_role_is_a_name", "set": { "sourceRole": "\u180e" }, "outcome": "admitted" },
+      { "name": "padded_name_is_a_name", "set": { "schemaId": "\u00a0a\u2003", "tags": [" a"] }, "outcome": "admitted" },
+      { "name": "file_separator_schema_id_is_a_name", "set": { "schemaId": "\u001c" }, "outcome": "admitted" },
       { "name": "source_ids_and_tags_present_and_named", "set": { "sourceRuntimeId": "r", "sourceAgentId": "a", "sourceRole": "w", "tags": ["a"] }, "outcome": "admitted" },
       { "name": "source_ids_and_tags_nil", "set": { "sourceRuntimeId": null, "sourceAgentId": null, "sourceRole": null, "tags": null }, "outcome": "admitted" },
       { "name": "empty_tag_list", "set": { "tags": [] }, "outcome": "admitted" }

contracts/cultnet/interop/write-receipt.md

@@ -36,7 +36,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir
    - **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests.
    - **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table.
    - **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session.
-   - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is empty or only whitespace; on a put's document, also a `storedAt` that is empty or only whitespace, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and empty or only whitespace, and a `tags` entry that is empty or only whitespace. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case.
+   - **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case.
    - **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`.
    - **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once.
    - Past the session (or a server's longer retention, if it keeps one) a resend is a new write.

packages/cultmesh-rs/tests/rudp_document_server.rs

@@ -3139,11 +3139,15 @@ fn unreadable_vector_text(vectors: &serde_json::Value) -> String {
 /// The put frame of a `blankFields` case: the valid document with the case's `set` applied. Serialized without the
 /// typed encoder, because that refuses what the case is about.
 fn blank_field_put(vectors: &serde_json::Value, set: &serde_json::Value) -> Vec<u8> {
+    blank_field_put_with_id(vectors, set, "m1")
+}
+
+fn blank_field_put_with_id(vectors: &serde_json::Value, set: &serde_json::Value, message_id: &str) -> Vec<u8> {
     #[derive(serde::Serialize)]
     #[serde(rename_all = "camelCase")]
     struct Put {
         schema_version: &'static str,
-        message_id: &'static str,
+        message_id: String,
         document: CultNetRawDocumentRecord,
     }
     let hex = set.get("payloadHex").and_then(|v| v.as_str()).unwrap_or(vectors["blankFields"]["validPutPayloadHex"].as_str().unwrap());
@@ -3160,7 +3164,7 @@ fn blank_field_put(vectors: &serde_json::Value, set: &serde_json::Value) -> Vec<
         source_role: optional("sourceRole"),
         tags: set.get("tags").and_then(|v| v.as_array()).map(|tags| tags.iter().map(|t| t.as_str().unwrap().to_string()).collect()),
     };
-    rmp_serde::to_vec_named(&Put { schema_version: "cultnet.document_put_raw.v0", message_id: "m1", document }).unwrap()
+    rmp_serde::to_vec_named(&Put { schema_version: "cultnet.document_put_raw.v0", message_id: message_id.to_string(), document }).unwrap()
 }
 
 /// Each `blankFields` case: a receipt-capable connection is answered with the admitted receipt, or with the one fixed
@@ -3188,8 +3192,8 @@ fn a_put_that_names_nothing_is_refused_as_unreadable_naming_the_write_and_a_name
                     vec![CultNetMessage::WriteAdmitted {
                         message_id: "m1".into(),
                         operation: "put".into(),
-                        schema_id: "note.v0".into(),
-                        record_key: "note:1".into(),
+                        schema_id: case["set"]["schemaId"].as_str().unwrap_or("note.v0").into(),
+                        record_key: case["set"]["recordKey"].as_str().unwrap_or("note:1").into(),
                     }],
                     "{name}"
                 );
@@ -3207,6 +3211,54 @@ fn a_put_that_names_nothing_is_refused_as_unreadable_naming_the_write_and_a_name
     Ok(())
 }
 
+fn code_point_string(hex: &str, times: usize) -> String {
+    std::char::from_u32(u32::from_str_radix(hex, 16).unwrap()).unwrap().to_string().repeat(times)
+}
+
+/// The Unicode White_Space set is the one blank rule: every listed code point, alone or repeated, blanks every field
+/// the rule covers; U+200B, U+FEFF, U+180E and the other listed non-White_Space code points name something.
+#[test]
+fn blank_means_exactly_the_unicode_white_space_code_points_in_every_field() -> Result<()> {
+    let vectors = vectors();
+    let fixed = unreadable_vector_text(&vectors);
+    let fields = ["schemaId", "recordKey", "storedAt", "sourceRuntimeId", "sourceAgentId", "sourceRole", "tags"];
+    let space = vectors["blankFields"]["whiteSpaceCodePoints"].as_array().unwrap();
+    let not_space = vectors["blankFields"]["notWhiteSpaceCodePoints"].as_array().unwrap();
+    assert_eq!(space.len(), 25);
+    for (index, (hex, blank)) in space.iter().map(|h| (h, true)).chain(not_space.iter().map(|h| (h, false))).enumerate() {
+        let hex = hex.as_str().unwrap();
+        for (field_index, field) in fields.iter().enumerate() {
+            // A session that is sent refusals is allowed to end, so each code point and field gets its own.
+            let sink = Sink::default();
+            let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
+            let mut peer = client(server.local_addr()?, 600 + (index * fields.len() + field_index) as u32)?;
+            connect(&mut server, &mut [&mut peer])?;
+            send_reliable(&mut peer, &receipt_hello())?;
+            assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![server_hello()]);
+            let mut admitted = 0;
+            for times in [1, 3] {
+                let text = code_point_string(hex, times);
+                let value = if *field == "tags" { serde_json::json!(["a", text]) } else { serde_json::json!(text) };
+                let set = serde_json::json!({ *field: value });
+                let id = format!("{hex}-{field}-{times}");
+                peer.send_reliable("schema", blank_field_put_with_id(&vectors, &set, &id))?;
+                let answers = frames_after(&mut server, &mut peer, 1)?;
+                let what = format!("U+{hex} in {field} x{times}");
+                assert!(!answers.is_empty(), "{what}: no answer");
+                if blank {
+                    assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some(id)), "{what}");
+                    assert_eq!(error_text_and_field(&answers[0]), (fixed.clone(), Some("message".into())), "{what}");
+                } else {
+                    assert!(matches!(answers[0], CultNetMessage::WriteAdmitted { .. }), "{what} is a name: {:?}", answers[0]);
+                    admitted += 1;
+                }
+            }
+            assert_eq!(sink.0.lock().unwrap().receipts.len(), admitted, "U+{hex} in {field}: only named writes are applied");
+        }
+    }
+    Ok(())
+}
+
 /// The same cases on a connection that never opted in: the same refusal as `error.v0`, the same text, nothing applied.
 #[test]
 fn a_put_that_names_nothing_is_refused_the_same_way_on_a_legacy_connection() -> Result<()> {

src/GameCult.Networking/CultNetDatabaseServer.cs

@@ -394,18 +394,26 @@ namespace GameCult.Networking
         // (write-receipt.md, "Required fields"), never handed to the store: a blank schemaId, recordKey or storedAt, an empty
         // payload, and a source id or tag that is present but blank.
         private static bool IsUnreadableWriteTarget(string? schemaId, string? recordKey) =>
-            string.IsNullOrWhiteSpace(schemaId) || string.IsNullOrWhiteSpace(recordKey);
+            IsBlank(schemaId) || IsBlank(recordKey);
 
         private static bool IsUnreadableDocument(CultNetRawDocumentRecord document) =>
             IsUnreadableWriteTarget(document.SchemaId, document.RecordKey)
-            || string.IsNullOrWhiteSpace(document.StoredAt)
+            || IsBlank(document.StoredAt)
             || document.Payload == null || document.Payload.Length == 0
             || IsPresentButBlank(document.SourceRuntimeId)
             || IsPresentButBlank(document.SourceAgentId)
             || IsPresentButBlank(document.SourceRole)
-            || document.Tags != null && document.Tags.Any(tag => string.IsNullOrWhiteSpace(tag));
+            || document.Tags != null && document.Tags.Any(IsBlank);
 
-        private static bool IsPresentButBlank(string? value) => value != null && string.IsNullOrWhiteSpace(value);
+        private static bool IsPresentButBlank(string? value) => value != null && IsBlank(value);
+
+        // Blank is the Unicode White_Space property, listed here and in write-receipt.md, not char.IsWhiteSpace: the platform's
+        // own test follows its Unicode tables (and has put U+180E in the set), the spec's does not move. A null is not text.
+        private static bool IsBlank(string? value) => value == null || value.All(IsWhiteSpaceCodePoint);
+
+        private static bool IsWhiteSpaceCodePoint(char c) =>
+            c is (>= '\u0009' and <= '\u000D') or '\u0020' or '\u0085' or '\u00A0' or '\u1680'
+                or (>= '\u2000' and <= '\u200A') or '\u2028' or '\u2029' or '\u202F' or '\u205F' or '\u3000';
 
         // A refusal: error.v1 carrying the write's id for a receipt-capable connection, the v0 error for any other.
         private static void Refuse(CultNetServerPeer peer, CultNetReceiptedWrite? write, CultNetErrorMessage error)

tests/GameCult.Networking.Tests/CultNetWriteReceiptServerTests.cs

@@ -1089,7 +1089,8 @@ namespace GameCult.Networking.Tests
             {
                 if (capable) AssertAdmitted(wire.Sent.Single(), "m1", "put", document.RecordKey);
                 else Assert.That(wire.Sent, Is.Empty, "a legacy connection is sent no receipt");
-                Assert.That(rig.TextOf(KeyA), Is.EqualTo("one"));
+                if (!vectors.GetProperty("set").EnumerateObject().Any(f => f.Name is "schemaId" or "recordKey"))
+                    Assert.That(rig.TextOf(KeyA), Is.EqualTo("one"));
                 return;
             }
 
@@ -1101,6 +1102,66 @@ namespace GameCult.Networking.Tests
             Assert.That(rig.TextOf(KeyA), Is.Null, "a write that names nothing is never applied");
         }
 
+        public static IEnumerable<string> WhiteSpaceCodePoints() =>
+            Vectors().GetProperty("blankFields").GetProperty("whiteSpaceCodePoints").EnumerateArray().Select(c => c.GetString()!);
+
+        public static IEnumerable<string> NonWhiteSpaceCodePoints() =>
+            Vectors().GetProperty("blankFields").GetProperty("notWhiteSpaceCodePoints").EnumerateArray().Select(c => c.GetString()!);
+
+        private static readonly string[] BlankRuleFields =
+            { "schemaId", "recordKey", "storedAt", "sourceRuntimeId", "sourceAgentId", "sourceRole", "tags" };
+
+        // Blank is the Unicode White_Space property in every runtime (write-receipt.md, Required fields), not whatever the
+        // platform's char.IsWhiteSpace says: each listed code point, alone or repeated, in each covered field is refused as
+        // unreadable and never applied; each listed non-White_Space code point (U+200B, U+FEFF, U+180E, ...) is a name.
+        private async Task AssertBlankRule(string hex, bool blank)
+        {
+            foreach (var field in BlankRuleFields)
+            foreach (var times in new[] { 1, 3 })
+            {
+                var text = string.Concat(Enumerable.Repeat(char.ConvertFromUtf32(Convert.ToInt32(hex, 16)), times));
+                var rig = Build();
+                var wire = new Wire();
+                await rig.Hello(wire, BothFrames);
+                wire.Sent.Clear();
+                var message = rig.PutOf("m1", KeyA, "one");
+                var document = message.Document!;
+                switch (field)
+                {
+                    case "schemaId": document.SchemaId = text; break;
+                    case "recordKey": document.RecordKey = text; break;
+                    case "storedAt": document.StoredAt = text; break;
+                    case "sourceRuntimeId": document.SourceRuntimeId = text; break;
+                    case "sourceAgentId": document.SourceAgentId = text; break;
+                    case "sourceRole": document.SourceRole = text; break;
+                    default: document.Tags = new[] { "a", text }; break;
+                }
+                var what = $"U+{hex} in {field} x{times}";
+
+                await rig.PutHandler(message, wire.Peer);
+
+                if (blank)
+                {
+                    var error = AssertRefused(wire.Sent.Single(), "m1", "message_invalid").ToV0();
+                    Assert.That(error.Details!.Field, Is.EqualTo("message"), what);
+                    Assert.That(error.Error, Is.EqualTo(UnreadableVectorText()), what);
+                    Assert.That(rig.TextOf(KeyA), Is.Null, $"{what}: never applied");
+                }
+                else
+                {
+                    AssertAdmitted(wire.Sent.Single(), "m1", "put", document.RecordKey);
+                }
+            }
+        }
+
+        [Test]
+        public async Task EveryUnicodeWhiteSpaceCodePointIsBlank([ValueSource(nameof(WhiteSpaceCodePoints))] string hex) =>
+            await AssertBlankRule(hex, blank: true);
+
+        [Test]
+        public async Task CodePointsOutsideWhiteSpaceNameSomething([ValueSource(nameof(NonWhiteSpaceCodePoints))] string hex) =>
+            await AssertBlankRule(hex, blank: false);
+
         // A nil payload names no document either; it is refused like an empty one, never handed to the store.
         [Test]
         public async Task APutWithANilPayloadIsRefusedAsUnreadable([Values] bool capable)