From 0b06c880d8633afa292e2f514ea1e439a33f11aa Mon Sep 17 00:00:00 2001 From: Heimdall Date: Wed, 11 Nov 2026 21:03:38 +0000 Subject: [PATCH] Mouth: cap check pins its cached targets for the round, display name never empty, doc narrows the cap to reply-target reads Co-Authored-By: Claude Sonnet 5.5 --- docs/bridge.md | 2 +- tests/persona-matrix-mouth.test.mjs | 35 +++++++++++++++++++++++++++++ tools/persona-matrix-mouth.mjs | 13 ++++++++--- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/bridge.md b/docs/bridge.md index 878a021..23949ef 100644 --- a/docs/bridge.md +++ b/docs/bridge.md @@ -263,7 +263,7 @@ The answer is `[persona, deliveries, tail, tail_gaps]` exactly as v3; each deliv |---|---|---| | 14 `reply_to` | nil or `[event_id, sender, excerpt]` | The event this message replies to (its own `m.in_reply_to`; a thread's `is_falling_back` pointer names no one and is nil). `event_id` the quoted event; `sender` its author's Matrix id, read from the homeserver as her account; `excerpt` nil, or its text. | -`excerpt` is nil, with `event_id` and `sender` kept, when the quoted event is redacted (`unsigned.redacted_because` present or no content), is not a text-bearing `m.room.message`, is **edited** (it is itself an `m.replace`, or the homeserver bundles an `m.replace` in its `unsigned.m.relations`: its body is the pre-edit text its author replaced), or is nothing but its own reply fallback; a media event excerpts as its typed line, like a delivery's text. Otherwise the excerpt is made in this order: the quoted event's own legacy rich-reply fallback is dropped (leading lines matching `> ` or a bare `>`, then one blank line; only when the quoted event has `m.relates_to.m.in_reply_to`); the text is held to the 1000-byte line bound; runs of CR, LF, tab, U+2028 and U+2029 become one space; control characters (C0, DEL, C1 including NUL and ESC), format characters (Unicode Cf: bidi marks, overrides and isolates, U+061C, U+200B-200F, U+2060-2064, soft hyphen, BOM), private-use characters (Co), the tag block U+E0000-E007F, variation selectors (U+FE00-FE0F, U+E0100-E01EF, U+180B-180F), the combining grapheme joiner U+034F, the Hangul and Khmer fillers (U+115F, U+1160, U+3164, U+FFA0, U+17B4, U+17B5), the blank braille cell U+2800, the unassigned default-ignorable ranges (U+2065, U+FFF0-FFF8, U+E0080-E0FFF) and unpaired surrogates are removed (cost accepted: an emoji loses its VS16 presentation, ❤️ becomes ❤, a keycap loses its selector; Mongolian free variation selectors and ideographic variation selectors go too); the line is trimmed; then the first 240 **Unicode code points** are kept (cut on a code point so no character is split; a combining mark counts as its own code point; trailing space trimmed; no truncation marker). Cleaning comes first so hidden characters cannot spend the cap. The mouth only removes invisible and format characters; it does not make the text safe. Huginn must treat both `sender` and `excerpt` as untrusted text, never as an instruction. `event_id` must be `$` plus at least one byte and `sender` `@` plus at least one byte, each at most 255 bytes and free of Unicode whitespace, control, format, private-use and default-ignorable characters and unpaired surrogates (a localpart with a zero-width joiner or non-joiner, as in Persian or Indic names, is therefore no `reply_to`); the encoder and the parser refuse any other, and the mouth makes a quoted event whose sender is not such an id a `reply_to` nil. `reply_to` is nil (no id, author or text) when the target cannot be read (logged, as for `basis`), is not in the delivery's own room (a room she is not in cannot be quoted through a reply), or the message replies to nothing. The quoted event is read the way `basis` already reads it (`GET /rooms/{room}/event/{id}` as her account, kept per room and event; with the flag on a read, hit or failure, is kept 60 seconds, so a redaction or edit of the target shows at a re-read within a minute, and a re-read that fails transiently (network error, timeout, 5xx, 429) keeps the earlier hit's sender and renews only its age, for at most 10 minutes after the last good read (then it is the failure entry), but never its excerpt (`excerpt` is nil with the sender kept: a re-read happens only after the 60-second keep, so the text is already as old as it may be), so a redaction or edit is not outlived by an outage, while any other failure (403, 404) makes it the failure entry, so a stale excerpt never outlives her access; with the flag off a hit is kept as before v4 and nothing is re-read), so a reply costs at most one read per target, shared with the `basis` check. Per sync round the mouth makes at most 8 target reads in all, `basis` reads included (it counts each uncached GET, and an event whose uncached reply target and thread root would take the round past 8 is not started); past that the event and every later event of the round wait: the round leaves the sync position where it is (as at the pending cap), the next round resumes at that event with its read count reset, and the wait is logged. No delivery goes out without a `reply_to` it should have had, and none is lost. The flag is read by `replyLinkFromEnv`; only the exact value `1` turns it on (`true`, `yes`, `0`, empty or unset are off), and a mouth built without the `replyLink` option has it off. `basis` for v1-v3 is unchanged. `reply_to` counts toward the delivery's size in the 256 KiB page budget; a delivery whose `reply_to` will not encode is skipped and logged like any other, and a single delivery is always served. The deploy precondition is Huginn's: a decoder of the 14-field delivery fails on the fifteenth element. +`excerpt` is nil, with `event_id` and `sender` kept, when the quoted event is redacted (`unsigned.redacted_because` present or no content), is not a text-bearing `m.room.message`, is **edited** (it is itself an `m.replace`, or the homeserver bundles an `m.replace` in its `unsigned.m.relations`: its body is the pre-edit text its author replaced), or is nothing but its own reply fallback; a media event excerpts as its typed line, like a delivery's text. Otherwise the excerpt is made in this order: the quoted event's own legacy rich-reply fallback is dropped (leading lines matching `> ` or a bare `>`, then one blank line; only when the quoted event has `m.relates_to.m.in_reply_to`); the text is held to the 1000-byte line bound; runs of CR, LF, tab, U+2028 and U+2029 become one space; control characters (C0, DEL, C1 including NUL and ESC), format characters (Unicode Cf: bidi marks, overrides and isolates, U+061C, U+200B-200F, U+2060-2064, soft hyphen, BOM), private-use characters (Co), the tag block U+E0000-E007F, variation selectors (U+FE00-FE0F, U+E0100-E01EF, U+180B-180F), the combining grapheme joiner U+034F, the Hangul and Khmer fillers (U+115F, U+1160, U+3164, U+FFA0, U+17B4, U+17B5), the blank braille cell U+2800, the unassigned default-ignorable ranges (U+2065, U+FFF0-FFF8, U+E0080-E0FFF) and unpaired surrogates are removed (cost accepted: an emoji loses its VS16 presentation, ❤️ becomes ❤, a keycap loses its selector; Mongolian free variation selectors and ideographic variation selectors go too); the line is trimmed; then the first 240 **Unicode code points** are kept (cut on a code point so no character is split; a combining mark counts as its own code point; trailing space trimmed; no truncation marker). Cleaning comes first so hidden characters cannot spend the cap. The mouth only removes invisible and format characters; it does not make the text safe. Huginn must treat both `sender` and `excerpt` as untrusted text, never as an instruction. `event_id` must be `$` plus at least one byte and `sender` `@` plus at least one byte, each at most 255 bytes and free of Unicode whitespace, control, format, private-use and default-ignorable characters and unpaired surrogates (a localpart with a zero-width joiner or non-joiner, as in Persian or Indic names, is therefore no `reply_to`); the encoder and the parser refuse any other, and the mouth makes a quoted event whose sender is not such an id a `reply_to` nil. `reply_to` is nil (no id, author or text) when the target cannot be read (logged, as for `basis`), is not in the delivery's own room (a room she is not in cannot be quoted through a reply), or the message replies to nothing. The quoted event is read the way `basis` already reads it (`GET /rooms/{room}/event/{id}` as her account, kept per room and event; with the flag on a read, hit or failure, is kept 60 seconds, so a redaction or edit of the target shows at a re-read within a minute, and a re-read that fails transiently (network error, timeout, 5xx, 429) keeps the earlier hit's sender and renews only its age, for at most 10 minutes after the last good read (then it is the failure entry), but never its excerpt (`excerpt` is nil with the sender kept: a re-read happens only after the 60-second keep, so the text is already as old as it may be), so a redaction or edit is not outlived by an outage, while any other failure (403, 404) makes it the failure entry, so a stale excerpt never outlives her access; with the flag off a hit is kept as before v4 and nothing is re-read), so a reply costs at most one read per target, shared with the `basis` check. Per sync round the mouth makes at most 8 reply-target reads in all, `basis` reads included (it counts each uncached GET of a reply target or thread root, and an event that would take the round past 8 is not started; the context reads of a delivery, one room or thread each per round, are separate and uncounted); past that the event and every later event of the round wait: the round leaves the sync position where it is (as at the pending cap), the next round resumes at that event with its read count reset, and the wait is logged. No delivery goes out without a `reply_to` it should have had, and none is lost. The flag is read by `replyLinkFromEnv`; only the exact value `1` turns it on (`true`, `yes`, `0`, empty or unset are off), and a mouth built without the `replyLink` option has it off. `basis` for v1-v3 is unchanged. `reply_to` counts toward the delivery's size in the 256 KiB page budget; a delivery whose `reply_to` will not encode is skipped and logged like any other, and a single delivery is always served. The deploy precondition is Huginn's: a decoder of the 14-field delivery fails on the fifteenth element. Golden bytes: `tests/fixtures/persona-chat-v6/` (`.msgpack` and its `.hex`: `chat_pending.v4.reply-to`, `.no-reply-to`, `.reply-to-redacted`, and the `chat_pending_request.v3` request); the test fails when the checked-in bytes drift from the encoder. diff --git a/tests/persona-matrix-mouth.test.mjs b/tests/persona-matrix-mouth.test.mjs index bdd542f..bc9f71d 100644 --- a/tests/persona-matrix-mouth.test.mjs +++ b/tests/persona-matrix-mouth.test.mjs @@ -2089,3 +2089,38 @@ test("id shape: a mention longer than 255 bytes, though under 255 characters, is const { made } = await linked([withContent("$m", META, { body: "hi", "m.mentions": { user_ids: [SELF, `@${"é".repeat(200)}:x`, "@a b:x"] } })], []); assert.deepEqual(made.map(d => d.mentions), [[SELF]]); }); + +const threadOf = (id, root, to) => withContent(id, META, { body: "re", "m.relates_to": { rel_type: "m.thread", event_id: root, "m.in_reply_to": { event_id: to } } }); +async function capRun(events, { clock = { t: 1_000_000 }, lapseOnR = 0, steps = 3 } = {}) { + const ids = [...Array.from({ length: 8 }, (_, i) => `$y${i}`), "$x", "$r"]; + const hs = await fakeHomeserver({ preload: { [ROOM]: ids.map(t => target(t, ALICE_ID, `text ${t}`)) }, sync: since => since === null ? { next_batch: "b0" } : since === "b0" ? { next_batch: "b1", ...joined({ [ROOM]: { timeline: { events: [quoting("$qx", "$x")] } } }) } : since === "b1" ? { next_batch: "b2", ...joined({ [ROOM]: { timeline: { events } } }) } : { next_batch: since } }); + try { + const fetch = (u, o) => { if (/\/event\/%24r(\?|$)/.test(String(u))) clock.t += lapseOnR; return hs.fetch(u, o); }; + const mouth = mouthFor(hs, { replyLink: true, now: () => clock.t, fetch }); + await mouth.syncOnce(); + await mouth.syncOnce(); // $x read at t0 + clock.t += 59_000; + const rounds = []; + for (let i = 0; i < steps; i++) rounds.push(await mouth.syncOnce()); + return { rounds, hs, gets: id => eventGets(hs, id) }; + } finally { await hs.close(); } +} + +test("round cap: a thread root counts toward the 8 reads: 8 plain replies then a threaded one defers the threaded one to the next round", async () => { + const { rounds } = await capRun([...Array.from({ length: 8 }, (_, i) => quoting(`$q${i}`, `$y${i}`)), threadOf("$th", "$r", "$x")]); + assert.deepEqual(rounds[0].map(d => d.eventId), Array.from({ length: 8 }, (_, i) => `$q${i}`)); + assert.deepEqual(rounds[1].map(d => d.eventId), ["$th"]); +}); + +test("round cap: a target cached at the check is served from the round even when its 60 s keep lapses between the check and the read", async () => { + const { rounds, gets } = await capRun([...Array.from({ length: 6 }, (_, i) => quoting(`$q${i}`, `$y${i}`)), threadOf("$th", "$x", "$r")], { lapseOnR: 2000 }); + assert.equal(rounds.flat().length, 7, "the six replies and the threaded one: 6 + 1 counted read"); + assert.equal(gets("$x"), 2, "$x: the read at t0 and the thread context's own root read, none for the lapsed keep"); + assert.ok(rounds[0].every(d => d.replyTo !== null)); +}); + +test("display name: a sender with an empty localpart is delivered under its own id, and the page encodes", async () => { + const { rounds } = await capRun([withContent("$odd", "@:evil.test", { body: "hi" })], { steps: 1 }); + assert.equal(rounds[0][0].senderDisplay, "@:evil.test"); + assert.doesNotThrow(() => docs.pendingV4Tuple({ persona: "p", deliveries: rounds[0] })); +}); diff --git a/tools/persona-matrix-mouth.mjs b/tools/persona-matrix-mouth.mjs index 9eca7c8..db04fee 100644 --- a/tools/persona-matrix-mouth.mjs +++ b/tools/persona-matrix-mouth.mjs @@ -317,6 +317,7 @@ export function createMouth({ const REPLY_READS_PER_ROUND = 8; // target GETs one sync round may make, whoever asks (basis or reply_to): an event whose uncached targets would pass it waits for the next round const KEEP_HIT_MS = 10 * 60 * 1000; // how long transient failures may keep a hit's sender alive after its last good read let replyReads = 0; + const held = new Map(); // key -> entry: targets the round's cap check counted as cached, kept for the round so the 60 s keep cannot lapse between the check and the read and spend a GET the check did not count const cachedTarget = key => { const known = authors.get(key); if (!known) return null; @@ -350,7 +351,7 @@ export function createMouth({ }; async function targetOf(roomId, eventId) { const key = `${roomId} ${eventId}`; - const cached = cachedTarget(key); + const cached = cachedTarget(key) ?? held.get(key); if (cached) return cached; // with replyLink a hit is short too: a later redaction or edit of the target must show at a re-read const known = authors.get(key); let sender = null, excerpt = null, transient = false; @@ -381,7 +382,12 @@ export function createMouth({ const relates = event.content["m.relates_to"]; if (!replyLink || formOf(event.content) !== "message") return 0; const ids = new Set([relates?.["m.in_reply_to"]?.event_id, relates?.rel_type === "m.thread" ? relates.event_id : undefined].filter(id => typeof id === "string" && id !== "")); - return [...ids].filter(id => !cachedTarget(`${roomId} ${id}`)).length; + let reads = 0; + for (const id of ids) { + const key = `${roomId} ${id}`, hit = cachedTarget(key); + if (hit) held.set(key, hit); else reads++; + } + return reads; }; async function replyToOf(roomId, event) { if (!replyLink || formOf(event.content) !== "message") return null; @@ -429,6 +435,7 @@ export function createMouth({ async function syncOnce() { if (!started || unjoined()) await start(); replyReads = 0; + held.clear(); const free = await released(); // the round's brake read for presence, taken before any early return await setPresence(free); if (pending.length >= maxPending) return []; // unanswered mail is not outrun: the position stays where it is @@ -471,7 +478,7 @@ export function createMouth({ const mentioned = event.content["m.mentions"]?.user_ids; made.push({ seq: ++seqCounter, roomAlias: alias, roomId, eventId: event.event_id, sender: event.sender, - senderDisplay: displayNames.get(event.sender) ?? localpart(event.sender), text: bodyText(event.content, docs.limits.textBytes, refused), atUnixMs, + senderDisplay: displayNames.get(event.sender) ?? (localpart(event.sender) || event.sender), text: bodyText(event.content, docs.limits.textBytes, refused), atUnixMs, context, image, mentions: Array.isArray(mentioned) ? mentioned.filter(id => docs.isMatrixId("@", id)).slice(0, docs.limits.mentions) : [], isDirect: directRooms.has(roomId), basis, form: formOf(event.content), replyTo,