Mouth: fetch one named event's image for its Persona, store it, post an inert receipt (CHAT_FETCH, off by default) #73

Merged
heimdall merged 4 commits from eureka/mouth-fetch-attachment into main 2026-11-12 00:59:28 +00:00
18 changed files with 937 additions and 10 deletions

View file

@ -298,6 +298,10 @@ The intended shape:
VoidBot still owns room reading, moderation judgment, archive/source retrieval, and Persona cognition. Bifrost owns Discord when Discord is being used as the Labor Platform interface or swarm transport.
### Fetch a named file (`chat_fetch_request.v1`, `chat_fetch_receipt.v1`)
A Persona's Mind shell holds no Matrix token, so the mouth keeps a file for her: `chat_fetch_request.v1` is `[request_id, persona, room, event_id]`, a room (alias or id) and an event id, nothing else. The mouth reads her live membership and the room's full state, reads the event itself and takes `content.url` from it, fetches it as her (5 MiB, magic bytes decide the type), compares the file event's `info.sha256` with the hash it computed, stores the bytes in her image store, and answers `chat_fetch_receipt.v1` `[request_id, outcome, room_id, image, receipt]` with one of 14 fixed outcome words. A file with no `info.sha256` is `StoredUnverified` and gets no receipt; a present hash that is not the computed one is `HashMismatch` and stores nothing. For a stored, hash-matching file the mouth posts the contract's `org.gamecult.file_received` event (`{event_id, sha256, size}`) as her own Matrix user, once per (file, Mind): a reply in the album thread, else a plain event with an `m.reference` to the file. The Matrix sender is the Mind; no content key names her. Idempotence is read from Matrix, not kept. Bridged and encrypted rooms and other people's rooms are refused. The kind is served only with `CHAT_FETCH=1` (exactly that value); off, it is answered as any unknown kind (`UnknownKind`). One 15 s deadline owns each fetch, and one Persona may ask 30 in an hour. Each fetch writes one log line, `fetch persona=... outcome=... receipt=...`, and a failure adds a trailing ` cause=<Class>/<status>`, on a thrown fetch failure, a store write that throws, or a receipt not proven: `Class` is the error's class name when it matches `[A-Za-z]{1,32}`, else `Error`; `status` is the HTTP status integer of a failed Matrix call, else `-`. No error-derived text (message, code, path, URL) reaches the log; the room and event ids, already shape-checked, are the only ids on the fetch line. `isFetchId` refuses a room-version-3 event id that contains `/` (standard base64), a fail-closed side: it is answered `NoAttachment`.
## Heimdall Integration Target
The correct future credential path is:

View file

@ -0,0 +1 @@
95a67265712d6631ab42726964676564526f6f6dc0c0c0

View file

@ -0,0 +1 @@
•¦req-f1«BridgedRoomÀÀÀ

View file

@ -0,0 +1 @@
95a67265712d6631a653746f726564b1216162633a67616d6563756c742e6f726794d985636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463642f616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261622e706e67a9696d6167652f706e67cd0800d94061626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162a84578697374696e67

View file

@ -0,0 +1 @@
95a67265712d6631a653746f726564b1216162633a67616d6563756c742e6f726794d985636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463642f616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261622e706e67a9696d6167652f706e67cd0800d94061626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162c0

View file

@ -0,0 +1 @@
95a67265712d6631a653746f726564b1216162633a67616d6563756c742e6f726794d985636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463642f616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261622e706e67a9696d6167652f706e67cd0800d94061626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162a6506f73746564

View file

@ -0,0 +1 @@
95a67265712d6631b053746f726564556e7665726966696564b1216162633a67616d6563756c742e6f726794d985636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463646364636463642f616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261622e706e67a9696d6167652f706e67cd0800d94061626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162616261626162c0

View file

@ -0,0 +1 @@
94a67265712d6631a6657572656b61b323737761726d3a67616d6563756c742e6f7267a62466696c6531

View file

@ -0,0 +1 @@
”¦req-f1¦eureka³#swarm:gamecult.org¦$file1

View file

@ -0,0 +1,676 @@
// A Persona asks her mouth to keep one named Matrix file (gamecult.persona.chat_fetch_request.v1): the mouth reads the event itself,
// checks it, stores the bytes and posts the contract's inert receipt as her. The homeserver here is an in-process fetch port that keeps
// room state, events, relations and idempotent sends the way Synapse does, records every call, and can stall a call until the caller's
// own signal aborts it.
import test from "node:test";
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFileSync, readdirSync } from "node:fs";
import { resolve } from "node:path";
import { createMouth, createRoomGate, fetchFromEnv } from "../tools/persona-matrix-mouth.mjs";
import { createImageStore, MAX_BYTES } from "../tools/persona-chat-images.mjs";
import * as docs from "../tools/persona-chat-documents.mjs";
import { png, jpeg, svg } from "./image-bytes.mjs";
import { personaStoreRoot } from "./image-bytes.mjs";
const HS = "gamecult.org";
const ROOM = "!room1:gamecult.org";
const ALIAS = `#swarm:${HS}`;
const ME = `@eureka:${HS}`;
const OTHER = `@kiko:${HS}`;
const FILE = "$file1";
const ROOT = "$album1";
const TOKEN = "tok-secret-eureka";
const sha = bytes => createHash("sha256").update(bytes).digest("hex");
const PLATFORM = ["disc", "ord"].join("");
const IMG = png(8, 8);
const IMG_SHA = sha(IMG);
const thread = { rel_type: "m.thread", event_id: ROOT };
const fileContent = (extra = {}) => ({ msgtype: "m.image", body: "cat.png", url: "mxc://gamecult.org/abc", info: { sha256: IMG_SHA, mimetype: "image/png", size: IMG.length }, "m.relates_to": thread, ...extra });
// holds: [{ match: /regex of "METHOD path"/, late: bool }]: the call waits until released or the caller's signal aborts. late: after its effect.
function synapse({ joined = [ROOM], state = [], events = {}, media = { abc: { bytes: IMG } }, holds = [], failing = {} } = {}) {
const s = { calls: [], raw: [], joined, state, events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent() }, ...events }, media, holds, failing, sent: new Map(), relations: new Map(), mediaHits: 0 };
const relationKey = (target, rel) => `${target}|${rel}`;
const respond = (status, body) => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
const stall = (hold, signal) => new Promise((ok, fail) => {
hold.open = ok;
signal?.addEventListener("abort", () => fail(signal.reason), { once: true });
});
s.fetch = async (url, init = {}) => {
const u = new URL(url);
const method = init.method ?? "GET";
const path = decodeURIComponent(u.pathname).replace(/^\/_matrix\/client\/v[13]/, "");
const key = `${method} ${path}`;
s.calls.push(`${key}${u.search}`);
s.raw.push(`${u.pathname}${u.search}`);
const hold = s.holds.find(h => h.match.test(key));
if (hold && !hold.late) await stall(hold, init.signal);
for (const [pattern, status] of Object.entries(s.failing)) if (new RegExp(pattern).test(key)) return respond(status, { errcode: "M_UNKNOWN" });
const finish = async response => { if (hold?.late) await stall(hold, init.signal); return response; };
if (path === "/account/whoami") return respond(200, { user_id: ME });
if (path.includes("/account_data/m.direct")) return respond(404, { errcode: "M_NOT_FOUND" });
if (path.startsWith("/directory/room/")) return respond(200, { room_id: ROOM });
if (path === "/joined_rooms") return respond(200, { joined_rooms: s.joined });
if (path === `/rooms/${ROOM}/state`) return respond(200, s.state);
const event = path.match(/^\/rooms\/([^/]+)\/event\/(.+)$/);
if (event) return s.events[event[2]] ? respond(200, s.events[event[2]]) : respond(404, { errcode: "M_NOT_FOUND" });
const rel = path.match(/^\/rooms\/([^/]+)\/relations\/([^/]+)\/([^/]+)\/(.+)$/);
if (rel) {
const pages = s.relations.get(relationKey(rel[2], rel[3])) ?? [[]];
const at = u.searchParams.get("from") === null ? 0 : Number(u.searchParams.get("from"));
return respond(200, { chunk: pages[at] ?? [], ...(at + 1 < pages.length ? { next_batch: String(at + 1) } : {}) });
}
const put = path.match(/^\/rooms\/([^/]+)\/send\/([^/]+)\/(.+)$/);
if (put && method === "PUT") {
const body = JSON.parse(init.body);
if (!s.sent.has(put[3])) {
const event = { type: put[2], event_id: `$sent${s.sent.size + 1}`, sender: ME, content: body };
s.sent.set(put[3], event);
const relates = body["m.relates_to"];
if (relates) {
const k = relationKey(relates.event_id, relates.rel_type);
s.relations.set(k, [[event, ...(s.relations.get(k)?.[0] ?? [])]]);
}
}
return finish(respond(200, { event_id: s.sent.get(put[3]).event_id }));
}
const dl = path.match(/^\/media\/download\/[^/]+\/(.+)$/);
if (dl) {
s.mediaHits++;
if (init.redirect === "error" && s.media[dl[1]]?.redirect) throw new TypeError("redirect");
const m = s.media[dl[1]];
if (!m) return respond(404, {});
return new Response(m.bytes, { status: m.status ?? 200, headers: m.headers ?? {} });
}
return respond(404, { errcode: "M_NOT_FOUND" });
};
s.receipts = () => [...s.sent.values()].filter(e => e.type === docs.fileReceivedType);
s.gets = pattern => s.calls.filter(c => pattern.test(c));
return s;
}
function mouthOn(s, { brake = async () => true, deadline = 5000, store = true, clock, gate, put } = {}) {
const root = personaStoreRoot();
const real = createImageStore({ root });
const puts = [];
const images = store ? { put: (...a) => { puts.push(a); return put ? put(...a) : real.put(...a); } } : null;
const logs = [];
const mouth = createMouth({
persona: "eureka", homeserver: "http://hs.test", serverName: HS, rooms: [ALIAS], token: TOKEN, fetch: s.fetch, state: { load: async () => null, save: async () => {} },
brake, log: l => logs.push(l), images, fetchDeadlineMs: deadline, ...(clock ? { now: clock } : {}), ...(gate ? { gate } : {}),
});
const files = () => readdirSync(root, { recursive: true }).filter(f => /\.(png|jpg|gif|webp)$/.test(f));
const ask = (over = {}) => mouth.fetchAttachment({ requestId: "r1", persona: "eureka", room: ALIAS, eventId: FILE, ...over });
return { mouth, ask, puts, logs, files, root, fetchLines: () => logs.filter(l => l.startsWith("fetch persona=")) };
}
const noSynapse = s => assert.deepEqual(s.calls, [], "no Synapse call was made");
test("PersonaMismatch and Brake answer before any Synapse call; a null image store is StoreFailed before any call", async () => {
const s = synapse();
assert.equal((await mouthOn(s).ask({ persona: "nova" })).outcome, "PersonaMismatch");
assert.equal((await mouthOn(s, { brake: async () => false }).ask()).outcome, "Brake");
assert.equal((await mouthOn(s, { store: false }).ask()).outcome, "StoreFailed");
noSynapse(s);
});
test("the request's ids are bounded: a room or event id that is not a path-safe Matrix id is refused with no Synapse request built from it", async () => {
const s = synapse();
const m = mouthOn(s);
for (const eventId of ["file1", "$", "$a b", "$a/../../x", "$a?x=1", "$a#f", "$a%2F", "$..", "$a\\b", `$${"x".repeat(300)}`, "$a\u0085b"])
assert.equal((await m.ask({ eventId })).outcome, "NoAttachment", JSON.stringify(eventId).slice(0, 30));
for (const room of ["swarm", "!", "!r/../x:y", "!r?x:y", "!a b:y", "#a/b:y", "!r%2F:y", `!${"x".repeat(300)}`])
assert.equal((await m.ask({ room })).outcome, "NotAMember", JSON.stringify(room).slice(0, 30));
noSynapse(s);
});
test("a hostile album root or event id in the file event's own m.relates_to is refused: no request is built from it and nothing is posted", async () => {
for (const root of ["$a/../../../etc", "$a?x=1", "$a#x", "$a%2F..", "not-an-event", "$a b", "$..", "", 7, null, "$a‮b"]) {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ "m.relates_to": { rel_type: "m.thread", event_id: root } }) } } });
const m = mouthOn(s);
const receipt = await m.ask();
assert.equal(receipt.outcome, "NoAttachment", `root ${JSON.stringify(root)}`);
assert.equal(s.gets(/relations/).length, 0, "no relations request was built from the root");
assert.equal(s.calls.filter(c => c.startsWith("PUT")).length, 0, "nothing posted");
assert.equal(s.mediaHits, 0);
assert.equal(m.puts.length, 0);
assert.ok(!s.calls.some(c => /\.\.|etc|%2F/i.test(decodeURIComponent(c.split("?")[0]).replace(/\/_matrix\/client\/v\d/, "")) && /relations|send/.test(c)), "no request path carries the root");
assert.ok(m.logs.every(l => !l.includes("/../") && !l.includes("x=1") && !l.includes("not-an-event") && !l.includes("\u202e")), "the hostile text is not logged");
}
});
test("the log carries no hostile id: a request id that fails the shape is logged as -", async () => {
const s = synapse();
const m = mouthOn(s);
await m.ask({ eventId: "$a\nfetch persona=eureka outcome=Stored" });
assert.equal(m.fetchLines().length, 1);
assert.match(m.fetchLines()[0], /event=- outcome=NoAttachment/);
});
test("fetch refuses a room the persona is not in, for an alias and for a bare id, before any state, event or media request", async () => {
const s = synapse({ joined: ["!other:gamecult.org"] });
const m = mouthOn(s);
assert.equal((await m.ask()).outcome, "NotAMember");
assert.equal((await m.ask({ room: "!dm:gamecult.org" })).outcome, "NotAMember");
assert.equal(s.gets(/\/state|\/event\/|media/).length, 0);
});
test("membership is read live, not from the cached sets", async () => {
const s = synapse();
const m = mouthOn(s);
assert.equal((await m.ask()).outcome, "Stored");
s.joined = []; // she left; the mouth's cached alias set still holds the room
s.calls.length = 0;
assert.equal((await m.ask({ requestId: "r2" })).outcome, "NotAMember");
assert.equal(s.gets(/\/state|\/event\/|media/).length, 0);
});
test("an unreadable /joined_rooms fails closed: FetchFailed and no state, event or media request", async () => {
const s = synapse({ failing: { "/joined_rooms": 500 } });
assert.equal((await mouthOn(s).ask()).outcome, "FetchFailed");
assert.equal(s.gets(/\/state|\/event\/|media/).length, 0);
});
test("a cold mouth starts before it resolves an alias and answers with the resolved room id; a start() that throws is FetchFailed", async () => {
const s = synapse();
const m = mouthOn(s);
const receipt = await m.ask();
assert.equal(receipt.roomId, ROOM);
assert.ok(receipt.image.path.startsWith(`${sha(ROOM)}/`), "the image path starts with sha256 of the resolved id");
const down = synapse({ failing: { "/account/whoami": 500 } });
assert.equal((await mouthOn(down).ask()).outcome, "FetchFailed");
});
test("a name speakableRoomId does not know is used as the room id, and the live /joined_rooms decides", async () => {
const s = synapse();
const m = mouthOn(s);
const receipt = await m.ask({ room: ROOM });
assert.equal(receipt.outcome, "Stored");
assert.equal(receipt.roomId, ROOM);
assert.ok(s.calls.every(c => !c.includes("null")), "no null id reaches a URL");
assert.equal((await m.ask({ room: "!unknown:gamecult.org" })).outcome, "NotAMember");
});
test("the room type is read from the full state list: a bridge event under any state key is BridgedRoom with no event request; lookalikes are not", async () => {
for (const type of ["m.bridge", "uk.half-shot.bridge"]) for (const content of [{ protocol: { id: "x" } }, {}]) {
const s = synapse({ state: [{ type, state_key: "net/chan-42", content }] });
assert.equal((await mouthOn(s).ask()).outcome, "BridgedRoom", `${type} ${JSON.stringify(content)}`);
assert.equal(s.gets(/\/event\//).length, 0);
}
for (const type of ["m.bridges", "org.example.bridge"]) assert.equal((await mouthOn(synapse({ state: [{ type, state_key: "", content: {} }] })).ask()).outcome, "Stored");
const s = synapse();
await mouthOn(s).ask();
assert.ok(s.calls.every(c => !/\/state\/(m|uk)\./.test(c)), "no by-type state read decides the room's type");
});
test("fetch refuses an encrypted room by state; both a bridge and encryption event give BridgedRoom", async () => {
const enc = { type: "m.room.encryption", state_key: "", content: {} };
const s = synapse({ state: [enc] });
assert.equal((await mouthOn(s).ask()).outcome, "TypeRefused");
assert.equal(s.gets(/\/event\//).length, 0);
assert.equal((await mouthOn(synapse({ state: [enc, { type: "m.bridge", state_key: "k", content: {} }] })).ask()).outcome, "BridgedRoom");
});
test("an unreadable room state fails closed, and the call order is joined_rooms, state, event, media", async () => {
for (const failing of [{ "/state": 500 }, { "/state": 404 }]) {
const s = synapse({ failing });
assert.equal((await mouthOn(s).ask()).outcome, "FetchFailed");
assert.equal(s.gets(/\/event\//).length, 0);
}
const s = synapse();
s.state = { not: "an array" };
assert.equal((await mouthOn(s).ask()).outcome, "FetchFailed");
const ok = synapse();
await mouthOn(ok).ask();
const order = ["/joined_rooms", "/state", "/event/", "media/download"].map(p => ok.calls.findIndex(c => c.includes(p)));
assert.deepEqual(order, [...order].sort((a, b) => a - b));
assert.ok(order.every(i => i >= 0));
});
test("an event of another room or none, or a 403 or 404 on it, is NotAMember", async () => {
for (const room_id of ["!elsewhere:gamecult.org", undefined]) {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id, sender: OTHER, content: fileContent() } } });
assert.equal((await mouthOn(s).ask()).outcome, "NotAMember");
assert.equal(s.mediaHits, 0);
}
assert.equal((await mouthOn(synapse({ failing: { "/event/": 403 } })).ask()).outcome, "NotAMember");
assert.equal((await mouthOn(synapse()).ask({ eventId: "$nothere" })).outcome, "NotAMember");
});
test("the event guards answer in their order and none makes a media request", async () => {
const cases = [
[{ type: "m.room.encrypted", content: { algorithm: "m.megolm.v1.aes-sha2" } }, "TypeRefused"],
[{ type: "m.room.message", content: { msgtype: "m.text", body: "hi" } }, "NoAttachment"],
[{ type: "m.room.message", content: {} }, "NoAttachment"],
[{ type: "m.room.message", content: fileContent({ "m.relates_to": { rel_type: "m.replace", event_id: "$x" } }) }, "NoAttachment"],
[{ type: "m.reaction", content: fileContent() }, "NoAttachment"],
[{ type: "m.room.message", content: fileContent({ file: { url: "mxc://gamecult.org/e", key: {} }, url: undefined }) }, "TypeRefused"],
[{ type: "m.room.message", content: fileContent({ msgtype: "m.audio" }) }, "TypeRefused"],
[{ type: "m.room.message", content: fileContent({ msgtype: "m.video" }) }, "TypeRefused"],
[{ type: "m.room.message", content: fileContent({ msgtype: "m.sticker" }) }, "NoAttachment"],
[{ type: "m.room.message", content: fileContent({ url: undefined }) }, "NoAttachment"],
[{ type: "m.room.message", content: fileContent({ url: 7 }) }, "NoAttachment"],
[{ type: "m.room.message", content: fileContent({ url: "https://evil.test/x" }) }, "NoAttachment"],
];
for (const [event, outcome] of cases) {
const s = synapse({ events: { [FILE]: { event_id: FILE, room_id: ROOM, sender: OTHER, ...event } } });
assert.equal((await mouthOn(s).ask()).outcome, outcome, JSON.stringify(event).slice(0, 80));
assert.equal(s.mediaHits, 0);
}
});
test("a platform puppet sender or platform-named content key is BridgedRoom with no media request; lookalikes are not refused", async () => {
const run = (sender, content) => { const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender, content } } }); return mouthOn(s).ask().then(r => [r.outcome, s.mediaHits]); };
for (const sender of [`@${PLATFORM}_123:${HS}`, `@_${PLATFORM}_1:${HS}`, `@${PLATFORM}bot:${HS}`, `@${PLATFORM}:${HS}`, `@x_${PLATFORM}:${HS}`.toUpperCase().replace(":", ":")])
assert.deepEqual(await run(sender, fileContent()), ["BridgedRoom", 0], sender);
assert.deepEqual(await run(OTHER, fileContent({ [`org.example.${PLATFORM}`]: {} })), ["BridgedRoom", 0]);
for (const sender of [`@${PLATFORM.replace("ord", "ourse")}:${HS}`, `@mediscord:${HS}`.replace("mediscord", "dis"), `@kiko:${HS}`])
assert.equal((await run(sender, fileContent()))[0], "Stored", sender);
});
test("the mxc comes from the event only; the media request names the event's own url", async () => {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ url: "mxc://gamecult.org/abc" }) } } });
await mouthOn(s).ask();
assert.deepEqual(s.raw.filter(p => p.includes("media")), ["/_matrix/client/v1/media/download/gamecult.org/abc"]);
assert.throws(() => docs.parseFetchRequest(["r", "eureka", ALIAS, FILE, "mxc://x/y"]), /4-field/);
assert.throws(() => docs.parseFetchRequest(["r", "eureka", ALIAS]), /4-field/);
});
test("a png under msgtype m.file with a text/html mimetype is stored as png; html bytes are TypeRefused", async () => {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ msgtype: "m.file", info: { sha256: IMG_SHA, mimetype: "text/html" } }) } } });
const r = await mouthOn(s).ask();
assert.equal(r.outcome, "Stored");
assert.equal(r.image.mime, "image/png");
for (const msgtype of ["m.image", "m.file"]) {
const html = Buffer.from("<html><script>alert(1)</script></html>");
const t = synapse({ media: { abc: { bytes: html } }, events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ msgtype, info: {} }) } } });
const m = mouthOn(t);
assert.equal((await m.ask()).outcome, "TypeRefused");
assert.equal(m.puts.length, 0);
}
});
test("every failure word of the image path maps to the enum", async () => {
const cases = [
[{ abc: { bytes: IMG, status: 500 } }, "FetchFailed"],
[{ abc: { bytes: Buffer.alloc(MAX_BYTES + 1, 1) } }, "TooLarge"],
[{ abc: { bytes: IMG, headers: { "content-length": String(MAX_BYTES + 1) } } }, "TooLarge"],
[{ abc: { bytes: png(10000, 10000) } }, "TooLarge"],
[{ abc: { bytes: png(0, 0) } }, "TypeRefused"],
[{ abc: { bytes: svg } }, "TypeRefused"],
[{ abc: { redirect: true, bytes: IMG } }, "FetchFailed"],
];
for (const [media, outcome] of cases) {
const m = mouthOn(synapse({ media }));
const r = await m.ask();
assert.equal(r.outcome, outcome);
assert.ok(docs.fetchOutcomes.includes(r.outcome));
assert.equal(m.puts.length, 0);
}
const unwritable = mouthOn(synapse());
unwritable.puts.length = 0;
const broken = createMouth({ persona: "eureka", homeserver: "http://hs.test", serverName: HS, rooms: [ALIAS], token: TOKEN, fetch: synapse().fetch, state: { load: async () => null, save: async () => {} }, brake: async () => true, images: { put: () => { throw Object.assign(new Error("x"), { code: "ENOSPC" }); } } });
assert.equal((await broken.fetchAttachment({ requestId: "r", persona: "eureka", room: ALIAS, eventId: FILE })).outcome, "StoreFailed");
});
test("an absent info.sha256 is StoredUnverified under the computed hash with no receipt, no relations read and no send", async () => {
for (const info of [undefined, null, "str", 7, [IMG_SHA], {}, { mimetype: "image/png" }]) {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ info }) } } });
const m = mouthOn(s);
const r = await m.ask();
assert.equal(r.outcome, "StoredUnverified", JSON.stringify(info));
assert.equal(r.receipt, null);
assert.equal(r.image.sha256, IMG_SHA);
assert.equal(s.gets(/relations/).length + s.calls.filter(c => c.startsWith("PUT")).length, 0);
assert.match(m.fetchLines()[0], new RegExp(`outcome=StoredUnverified size=${IMG.length} type=image/png sha256=${IMG_SHA} receipt=skipped`));
}
});
test("a present info.sha256 that is not the computed hash is a mismatch: nothing stored, nothing posted, the claim never logged", async () => {
const claims = [null, 7, {}, "", IMG_SHA.slice(1), `${IMG_SHA}0`, "z".repeat(64), IMG_SHA.toUpperCase(), sha("other"), ` ${IMG_SHA}`];
for (const claim of claims) {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ info: { sha256: claim } }) } } });
const m = mouthOn(s);
const r = await m.ask();
assert.equal(r.outcome, "HashMismatch", JSON.stringify(claim));
assert.equal(m.puts.length, 0);
assert.equal(m.files().length, 0);
assert.equal(s.calls.filter(c => c.startsWith("PUT")).length, 0);
assert.match(m.fetchLines()[0], new RegExp(`sha256=${IMG_SHA} `), "the computed hash, never the claim");
}
});
test("a matching info.sha256 is Stored; in an album thread the receipt is Posted and is the contract's, nothing more", async () => {
const s = synapse();
const m = mouthOn(s);
const r = await m.ask();
assert.equal(r.outcome, "Stored");
assert.equal(r.receipt, "Posted");
assert.equal(m.files().length, 1);
const [event] = s.receipts();
assert.equal(event.sender, ME);
assert.deepEqual(event.content, { "m.relates_to": { rel_type: "m.thread", event_id: ROOT }, [docs.fileReceivedType]: { event_id: FILE, sha256: IMG_SHA, size: IMG.length } });
const text = JSON.stringify(event.content);
assert.ok(!/mxc|\/|https?:|tok-secret|cat\.png|"mind"/.test(text.replace(/"m\.relates_to"|"rel_type"|"m\.thread"|"org\.gamecult\.file_received"/g, "")), "no path, scheme, mxc, token, filename or mind key");
assert.ok(m.logs.every(l => !l.includes(TOKEN) && !l.includes("mxc://")));
assert.match(m.fetchLines()[0], /receipt=posted/);
const txn = [...s.sent.keys()][0];
assert.equal(txn, `fr-${sha("eureka|" + FILE)}`);
});
const plainFile = relates => ({ [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: relates === undefined ? (({ "m.relates_to": _, ...rest }) => rest)(fileContent()) : fileContent({ "m.relates_to": relates }) } });
test("a file in no thread gets one plain receipt referencing the file; its reference and content.event_id are one value", async () => {
for (const relates of [undefined, { "m.in_reply_to": { event_id: "$q" } }, { rel_type: "m.reference", event_id: "$z" }]) {
const s = synapse({ events: plainFile(relates) });
const m = mouthOn(s);
const r = await m.ask({ room: ROOM });
assert.equal(r.receipt, "Posted");
assert.equal(s.receipts().length, 1);
const [event] = s.receipts();
assert.deepEqual(Object.keys(event.content).sort(), ["m.relates_to", docs.fileReceivedType]);
assert.deepEqual(event.content["m.relates_to"], { rel_type: "m.reference", event_id: FILE });
assert.deepEqual(Object.keys(event.content[docs.fileReceivedType]).sort(), ["event_id", "sha256", "size"]);
assert.equal(event.content["m.relates_to"].event_id, event.content[docs.fileReceivedType].event_id);
const again = await mouthOn(s).ask(); // a restarted mouth reads Matrix, keeps nothing
assert.equal(again.receipt, "Existing");
assert.equal(s.receipts().length, 1);
}
const root = synapse({ events: plainFile({ rel_type: "m.thread", event_id: FILE }) }); // the file is itself a thread root's reply to itself: still a valid thread shape
assert.equal((await mouthOn(root).ask()).receipt, "Posted");
});
test("idempotence reads the v1 relations path of the thread or of the file, follows next_batch for three pages and never a fourth", async () => {
const s = synapse();
s.relations.set(`${ROOT}|m.thread`, [[], [], [], [{ type: docs.fileReceivedType, sender: ME, content: { [docs.fileReceivedType]: { event_id: FILE } } }]]);
const m = mouthOn(s);
const r = await m.ask();
assert.equal(s.gets(/relations/).length, 3, "a fourth page is never asked");
assert.ok(s.raw.includes(`/_matrix/client/v1/rooms/${encodeURIComponent(ROOM)}/relations/${encodeURIComponent(ROOT)}/m.thread/${docs.fileReceivedType}?dir=b&limit=100`));
assert.ok(s.raw.includes(`/_matrix/client/v1/rooms/${encodeURIComponent(ROOM)}/relations/${encodeURIComponent(ROOT)}/m.thread/${docs.fileReceivedType}?dir=b&limit=100&from=1`), "page two via next_batch");
assert.ok(s.calls.every(c => !c.includes("event_type=")));
assert.equal(r.outcome, "Stored");
assert.equal(r.receipt, null, "not exhausted proves nothing: no post");
assert.equal(s.receipts().length, 0);
assert.match(m.fetchLines()[0], /receipt=failed/);
const t = synapse();
t.relations.set(`${ROOT}|m.thread`, [[], [{ type: docs.fileReceivedType, sender: ME, content: { [docs.fileReceivedType]: { event_id: FILE } } }]]);
assert.equal((await mouthOn(t).ask()).receipt, "Existing");
assert.equal(t.receipts().length, 0);
const f = synapse({ events: plainFile(undefined) });
await mouthOn(f).ask({ room: ROOM });
assert.ok(f.raw.some(p => p.startsWith(`/_matrix/client/v1/rooms/${encodeURIComponent(ROOM)}/relations/${encodeURIComponent(FILE)}/m.reference/${docs.fileReceivedType}?dir=b&limit=100`)));
});
test("a receipt by another sender, for another file, or redacted does not suppress hers, in both listings", async () => {
const mine = (over = {}) => ({ type: docs.fileReceivedType, sender: ME, content: { [docs.fileReceivedType]: { event_id: FILE } }, ...over });
const decoys = [mine({ sender: OTHER }), mine({ content: { [docs.fileReceivedType]: { event_id: "$other" } } }), mine({ unsigned: { redacted_because: {} }, content: {} })];
for (const [events, key, room] of [[undefined, `${ROOT}|m.thread`, ALIAS], [plainFile(undefined), `${FILE}|m.reference`, ROOM]]) {
const s = synapse(events ? { events } : {});
s.relations.set(key, [decoys]);
const r = await mouthOn(s).ask({ room });
assert.equal(r.receipt, "Posted");
assert.equal(s.receipts().length, 1);
}
});
test("a failed relations read or a failed post does not fail the fetch: Stored, receipt nil, file kept, log receipt=failed", async () => {
for (const [events, failing, room] of [[undefined, { "/relations/": 500 }, ALIAS], [undefined, { "PUT .*/send/": 500 }, ALIAS], [plainFile(undefined), { "/relations/": 500 }, ROOM], [plainFile(undefined), { "PUT .*/send/": 500 }, ROOM]]) {
const s = synapse({ ...(events ? { events } : {}), failing });
const m = mouthOn(s);
const r = await m.ask({ room });
assert.equal(r.outcome, "Stored");
assert.equal(r.receipt, null);
assert.equal(m.files().length, 1);
assert.match(m.fetchLines()[0], /receipt=failed/);
assert.equal(s.receipts().length, 0);
}
});
test("a brake placed after the file is stored posts no receipt: Stored, receipt nil, log receipt=skipped", async () => {
const s = synapse();
let reads = 0;
const m = mouthOn(s, { brake: async () => { reads++; return !s.mediaHits || reads <= 2 ? true : s.calls.filter(c => c.includes("relations")).length === 0 ? true : false; } });
const r = await m.ask();
assert.equal(r.outcome, "Stored");
assert.equal(r.receipt, null);
assert.equal(m.files().length, 1);
assert.equal(s.calls.filter(c => c.startsWith("PUT")).length, 0);
assert.match(m.fetchLines()[0], /receipt=skipped/);
});
test("the fetch is rate limited per Persona: the 31st in an hour is RateLimited with no request; every request counts", async () => {
const s = synapse({ joined: [] });
let t = 0;
const m = mouthOn(s, { clock: () => t });
for (let i = 0; i < 30; i++) assert.equal((await m.ask({ requestId: `r${i}` })).outcome, "NotAMember");
s.calls.length = 0;
assert.equal((await m.ask({ requestId: "r31" })).outcome, "RateLimited");
noSynapse(s);
t += 3_600_001;
assert.equal((await m.ask({ requestId: "r32" })).outcome, "NotAMember");
});
test("the fetch runs inside the room gate with a send: a fetch queued behind a slow holder whose deadline fires does no Synapse GET and stores nothing", async () => {
const s = synapse();
const gate = createRoomGate();
let release;
const holder = gate.run(ROOM, () => new Promise(ok => { release = ok; }));
const m = mouthOn(s, { gate, deadline: 80 });
const asked = await m.ask({ room: ROOM }); // answers at the deadline, while the holder still holds the gate
assert.equal(asked.outcome, "FetchFailed");
s.calls.length = 0;
release();
await holder;
await new Promise(r => setTimeout(r, 30));
assert.deepEqual(s.calls, [], "the queued fetch ran nothing past its deadline");
assert.equal(m.puts.length, 0);
assert.equal(m.files().length, 0);
assert.equal(m.fetchLines().length, 1, "one line per fetch");
assert.match(m.fetchLines()[0], /outcome=FetchFailed .*receipt=skipped/);
});
test("a brake placed while a fetch waits in the gate stops it before any request", async () => {
const s = synapse();
const gate = createRoomGate();
let release;
const holder = gate.run(ROOM, () => new Promise(ok => { release = ok; }));
let free = true;
const m = mouthOn(s, { gate, brake: async () => free });
await m.mouth.start();
s.calls.length = 0;
const asked = m.ask({ room: ROOM });
await new Promise(r => setTimeout(r, 20));
free = false;
release();
await holder;
assert.equal((await asked).outcome, "Brake");
assert.deepEqual(s.calls, []);
assert.equal(m.puts.length, 0);
});
test("one deadline covers the whole fetch: a stall in each read before the put is FetchFailed receipt=skipped, nothing is put or sent once it is released", async () => {
for (const match of [/GET \/joined_rooms/, /GET \/rooms\/[^/]+\/state$/, /GET \/rooms\/[^/]+\/event\//, /media\/download/]) {
const hold = { match };
const s = synapse();
const m = mouthOn(s, { deadline: 120 });
await m.mouth.start();
s.holds.push(hold);
const r = await m.ask({ room: ROOM });
assert.equal(r.outcome, "FetchFailed", String(match));
hold.open?.();
await new Promise(r => setTimeout(r, 20));
assert.equal(m.puts.length, 0);
assert.equal(s.calls.filter(c => c.startsWith("PUT")).length, 0);
assert.equal(m.fetchLines().length, 1);
assert.match(m.fetchLines()[0], /outcome=FetchFailed .*receipt=skipped/);
}
});
test("a deadline after the put leaves a stored file and no receipt, logged receipt=failed; the retry posts exactly one receipt", async () => {
for (const match of [/GET \/rooms\/[^/]+\/relations\//, /PUT \/rooms\/[^/]+\/send\//]) {
const hold = { match, late: /PUT/.test(match.source) };
const s = synapse({ holds: [hold] });
const m = mouthOn(s, { deadline: 150 });
const r = await m.ask({ room: ROOM });
assert.equal(r.outcome, "FetchFailed");
assert.equal(r.image, null);
assert.equal(m.puts.length, 1);
assert.equal(m.files().length, 1);
assert.match(m.fetchLines()[0], /outcome=FetchFailed .*receipt=failed/);
s.holds.length = 0;
const retry = await m.ask({ room: ROOM, requestId: "r2" });
assert.equal(retry.outcome, "Stored");
assert.ok(retry.receipt === "Posted" || retry.receipt === "Existing");
assert.equal(s.receipts().length, 1, "the same txn id: still one receipt");
}
});
test("the mouth log line carries the computed hash, the planted token and mxc are in no line, and a Stored fetch has exactly one line", async () => {
const s = synapse();
const m = mouthOn(s);
await m.ask();
assert.equal(m.fetchLines().length, 1);
assert.equal(m.fetchLines()[0], `fetch persona=eureka room=${ROOM} event=${FILE} outcome=Stored size=${IMG.length} type=image/png sha256=${IMG_SHA} receipt=posted`);
assert.ok(m.logs.every(l => !l.includes(TOKEN) && !l.includes("mxc")));
const j = mouthOn(synapse({ media: { abc: { bytes: jpeg(8, 8) } }, events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ info: { sha256: IMG_SHA } }) } } }));
assert.equal((await j.ask()).outcome, "HashMismatch");
assert.match(j.fetchLines()[0], /type=image\/jpeg/);
});
test("the receipt document follows its one table", () => {
const image = { path: `${sha(ROOM)}/${IMG_SHA}.png`, mime: "image/png", size: 100, sha256: IMG_SHA };
for (const receipt of [null, "Posted", "Existing"]) assert.doesNotThrow(() => docs.fetchReceiptTuple({ requestId: "r", outcome: "Stored", roomId: ROOM, image, receipt }));
assert.doesNotThrow(() => docs.fetchReceiptTuple({ requestId: "r", outcome: "StoredUnverified", roomId: ROOM, image }));
for (const receipt of ["Posted", "Existing", "$ev", "posted"]) assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome: "StoredUnverified", roomId: ROOM, image, receipt }));
assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome: "Stored", roomId: ROOM, image, receipt: "$ev" }));
assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome: "Stored" }), /exactly for Stored/);
for (const outcome of docs.fetchOutcomes.filter(w => !w.startsWith("Stored"))) {
assert.doesNotThrow(() => docs.fetchReceiptTuple({ requestId: "r", outcome }));
assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome, roomId: ROOM }));
assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome, image }));
assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome, receipt: "Posted" }));
}
assert.equal(docs.fetchOutcomes.length, 14);
assert.throws(() => docs.fetchReceiptTuple({ requestId: "r", outcome: "Nope" }), /unknown fetch outcome/);
const tuple = docs.fetchReceiptTuple({ requestId: "r", outcome: "Stored", roomId: ROOM, image, receipt: "Posted" });
assert.equal(tuple.length, 5);
assert.deepEqual(docs.parseFetchReceipt(tuple), { requestId: "r", outcome: "Stored", roomId: ROOM, image, receipt: "Posted" });
assert.throws(() => docs.parseFetchReceipt([...tuple, "extra"]), /5-field/);
assert.throws(() => docs.parseFetchReceipt(tuple.slice(0, 4)), /5-field/);
assert.throws(() => docs.parseFetchReceipt(["r", "StoredUnverified", ROOM, tuple[3], "Posted"]));
assert.throws(() => docs.parseFetchRequest(["r", "eureka", "x".repeat(300), FILE]), /over 255/);
});
test("the flag is exactly one", () => {
assert.equal(fetchFromEnv({}), false);
for (const v of ["0", "true", "yes", "", " 1", "1 ", "01"]) assert.equal(fetchFromEnv({ CHAT_FETCH: v }), false, JSON.stringify(v));
assert.equal(fetchFromEnv({ CHAT_FETCH: "1" }), true);
});
test("images.put has exactly two call sites in the mouth and the fetch's comes after the hash decision; no second media fetcher", () => {
const source = readFileSync(resolve(import.meta.dirname, "..", "tools", "persona-matrix-mouth.mjs"), "utf8");
assert.equal(source.match(/images\.put\(/g).length, 2);
const fetchBody = source.slice(source.indexOf("async function fetchAttachment"), source.indexOf("async function postFileReceipt"));
assert.ok(fetchBody.indexOf("HashMismatch") < fetchBody.indexOf("images.put("), "the hash decision precedes the put");
assert.ok(fetchBody.indexOf("throwIfAborted", fetchBody.indexOf("HashMismatch")) < fetchBody.indexOf("images.put("), "the deadline is checked right before the put");
assert.ok(!/media\/download/.test(source), "no media GET outside fetchImage");
assert.equal([...source.matchAll(/\/state\/(m\.|uk\.)|event_type=/g)].length, 0);
});
test("the receipt names the event the GET resolved: reference and content.event_id both carry the returned id, in a thread file and a plain one", async () => {
const answered = { type: "m.room.message", event_id: "$resolved", room_id: ROOM, sender: OTHER };
const plain = synapse({ events: { [FILE]: { ...answered, content: plainFile(undefined)[FILE].content } } });
assert.equal((await mouthOn(plain).ask({ room: ROOM })).receipt, "Posted");
const [event] = plain.receipts();
assert.equal(event.content["m.relates_to"].event_id, "$resolved");
assert.equal(event.content[docs.fileReceivedType].event_id, "$resolved");
const threaded = synapse({ events: { [FILE]: { ...answered, content: fileContent() } } });
await mouthOn(threaded).ask({ room: ROOM });
assert.equal(threaded.receipts()[0].content[docs.fileReceivedType].event_id, "$resolved");
const bad = synapse({ events: { [FILE]: { ...answered, event_id: "$a/../b", content: fileContent() } } });
assert.equal((await mouthOn(bad).ask({ room: ROOM })).outcome, "NoAttachment");
assert.equal(bad.receipts().length, 0);
});
test("a failure leaves the log as fixed words: a transport error holding a URL, an mxc and a sender-chosen 200-byte album root reaches no line", async () => {
const root = `$${"r".repeat(199)}`;
const taint = `http://hs.test/_matrix/client/v3/joined_rooms mxc://gamecult.org/abc ${root} Bearer ${TOKEN}`;
for (const [pattern, expectFetch] of [[/GET \/rooms\/[^/]+\/relations\//, "Stored"], [/GET \/rooms\/[^/]+\/state$/, "FetchFailed"], [/PUT \/rooms\/[^/]+\/send\//, "Stored"]]) {
const s = synapse({ events: { [FILE]: { type: "m.room.message", event_id: FILE, room_id: ROOM, sender: OTHER, content: fileContent({ "m.relates_to": { rel_type: "m.thread", event_id: root } }) } } });
const real = s.fetch;
s.fetch = async (url, init = {}) => {
if (pattern.test(`${init.method ?? "GET"} ${decodeURIComponent(new URL(url).pathname).replace(/^\/_matrix\/client\/v[13]/, "")}`)) throw new Error(`fetch failed ${taint}`);
return real(url, init);
};
const m = mouthOn(s);
assert.equal((await m.ask({ room: ROOM })).outcome, expectFetch, String(pattern));
assert.equal(m.fetchLines().length, 1, "still one fetch line");
for (const line of m.logs) {
for (const bit of ["http", "mxc", root, "r".repeat(40), TOKEN, "unreachable"]) assert.ok(!line.includes(bit), `${bit} in: ${line.slice(0, 120)}`);
}
assert.match(m.fetchLines()[0], / cause=Error\/-$/);
}
const status = synapse();
status.failing = { "/relations/": 500 };
const m = mouthOn(status);
await m.ask();
assert.match(m.fetchLines()[0], /receipt=failed cause=Error\/500$/);
});
test("the deadline aborts the in-flight media GET: the room gate is free again at once, not after the media timeout", async () => {
const s = synapse({ holds: [{ match: /media\/download/ }] });
const gate = createRoomGate();
const m = mouthOn(s, { gate, deadline: 100 });
assert.equal((await m.ask({ room: ROOM })).outcome, "FetchFailed");
const freed = await Promise.race([gate.run(ROOM, async () => "free"), new Promise(r => setTimeout(() => r("held"), 300))]);
assert.equal(freed, "free");
assert.equal(m.puts.length, 0);
});
test("a redacted receipt does not count as hers even when its content is intact", async () => {
const s = synapse();
s.relations.set(`${ROOT}|m.thread`, [[{ type: docs.fileReceivedType, sender: ME, unsigned: { redacted_because: {} }, content: { [docs.fileReceivedType]: { event_id: FILE } } }]]);
assert.equal((await mouthOn(s).ask({ room: ROOM })).receipt, "Posted");
assert.equal(s.receipts().length, 1);
});
test("a deadline that passes while the brake is re-read before the receipt PUT sends nothing", async () => {
const s = synapse();
let reads = 0, free;
const m = mouthOn(s, { deadline: 150, brake: async () => (++reads === 3 ? new Promise(ok => { free = () => ok(true); }) : true) });
assert.equal((await m.ask({ room: ROOM })).outcome, "FetchFailed");
free();
await new Promise(r => setTimeout(r, 30));
assert.equal(s.calls.filter(c => c.startsWith("PUT")).length, 0);
assert.equal(m.puts.length, 1);
});
test("a store that throws is one fetch line with a fixed cause: its code, message and name text reach no line", async () => {
const taint = `/srv/images/secret-path http://hs.test/x mxc://gamecult.org/abc ${TOKEN}`;
const s = synapse();
const m = mouthOn(s, { put: () => { throw Object.assign(new Error(taint), { code: taint, name: "not a class name" }); } });
assert.equal((await m.ask()).outcome, "StoreFailed");
assert.equal(m.fetchLines().length, 1);
assert.match(m.fetchLines()[0], /outcome=StoreFailed .* cause=Error\/-$/);
assert.equal(m.logs.length, m.fetchLines().length, "no other line");
for (const line of m.logs) for (const bit of ["secret-path", "http", "mxc", TOKEN, "EACCES"]) assert.ok(!line.includes(bit), `${bit} in: ${line}`);
assert.equal(s.calls.filter(c => c.startsWith("PUT")).length, 0, "no receipt for an unstored file");
});
test("an event id with characters encodeURIComponent changes goes out as one encoded path segment in every request built from it", async () => {
const hostile = ["$a+b=c", "$a@b:c", "$a&b", "$a b".replace(" ", "!"), "$a'(b)*", "$a;b,c", "$a$b"];
for (const [id, threaded] of hostile.flatMap(h => [[h, true], [h, false]])) {
assert.ok(docs.isFetchId("$", id), id);
assert.notEqual(encodeURIComponent(id), id, `${id} is changed by encoding`);
const s = synapse({ events: { [id]: { type: "m.room.message", event_id: id, room_id: ROOM, sender: OTHER, content: fileContent({ "m.relates_to": threaded ? { rel_type: "m.thread", event_id: id } : undefined }) } } });
const m = mouthOn(s);
assert.equal((await m.ask({ eventId: id, room: ROOM })).outcome, "Stored", id);
const enc = encodeURIComponent(id);
const eventGet = s.raw.filter(p => p.includes("/event/"));
const relGet = s.raw.filter(p => p.includes("/relations/"));
assert.ok(eventGet.length === 1 && eventGet[0] === `/_matrix/client/v3/rooms/${encodeURIComponent(ROOM)}/event/${enc}`, `event GET: ${eventGet}`);
assert.ok(relGet.length >= 1 && relGet.every(p => p.startsWith(`/_matrix/client/v1/rooms/${encodeURIComponent(ROOM)}/relations/${enc}/${threaded ? "m.thread" : "m.reference"}/`)), `relations GET: ${relGet}`);
assert.ok(s.raw.every(p => !p.split("?")[0].includes(id) || id === enc), "no raw id in a request path");
}
});

View file

@ -668,11 +668,11 @@ test("v4 paging: reply_to counts toward each delivery's size, and one oversize o
});
// Dial a throwaway endpoint, send one request and return the first document answered.
async function askOnce({ replyLink, kind, tuple }) {
const mouth = { pendingAfter: () => [{ ...v4Delivery, replyTo: quoted }], tail: async () => ({ lines: [], gaps: [], rooms: [] }), speakableRooms: () => [] };
async function askOnce({ replyLink, kind, tuple, fetchEnabled, mouth: given }) {
const mouth = given ?? { pendingAfter: () => [{ ...v4Delivery, replyTo: quoted }], tail: async () => ({ lines: [], gaps: [], rooms: [] }), speakableRooms: () => [] };
const key = "connection-key-v4";
const logs = [];
const server = await startChatEndpoint({ mouths: new Map([["eureka", mouth]]), runtime, port: 0, connectionKey: key, runtimeId: "mouth-v4", log: l => logs.push(l), ...(replyLink === undefined ? {} : { replyLink }) });
const server = await startChatEndpoint({ mouths: new Map([["eureka", mouth]]), runtime, port: 0, connectionKey: key, runtimeId: "mouth-v4", log: l => logs.push(l), ...(replyLink === undefined ? {} : { replyLink }), ...(fetchEnabled === undefined ? {} : { fetchEnabled }) });
const client = dgram.createSocket("udp4");
await new Promise(ok => client.bind(0, "127.0.0.1", ok));
const connection = new runtime.cultnet.CultNetRudpSocketTransportConnection({ runtimeId: "huginn-v4", socket: client, mode: "client", remoteHost: "127.0.0.1", remotePort: server.address.port, connectionId: connectionIdFor(key), initialSequence: 1, resendDelayMs: 50, resendPollMs: 10, maxFragmentBytes: 1200 });
@ -711,3 +711,57 @@ test("v4 flag on: the v3 request kind is answered with chat_pending.v4 over RUDP
const v1 = await askOnce({ replyLink: true, kind: docs.kinds.pendingRequest, tuple: request });
assert.equal(msgpack.decode(v1.document.payload)[1][0].length, 12);
});
// ---- chat_fetch (docs/bridge.md, "Fetch a named file"): the request and receipt kinds, behind CHAT_FETCH, default off.
const fetchTuple = docs.fetchRequestTuple({ requestId: "req-f1", persona: "eureka", room: "#swarm:gamecult.org", eventId: "$file1" });
const fetchStub = () => { const asked = []; return { asked, mouth: { fetchAttachment: async request => { asked.push(request); return { requestId: request.requestId, outcome: "NoAttachment", roomId: null, image: null, receipt: null }; } } }; };
test("fetch flag off (unset, false): a fetch request is answered exactly as any unknown kind is, byte for byte, with no mouth call and the existing unknown-kind log line only", async () => {
for (const fetchEnabled of [undefined, false]) {
const { asked, mouth } = fetchStub();
const off = await askOnce({ fetchEnabled, kind: docs.kinds.fetchRequest, tuple: fetchTuple, mouth });
assert.deepEqual([off.document.schemaId, docs.parseRefusal(msgpack.decode(off.document.payload))], [docs.kinds.refusal, { persona: "eureka", refusal: "UnknownKind" }]);
assert.equal(asked.length, 0, "the mouth was never asked");
assert.deepEqual(off.logs, [`chat endpoint refused a request of an unknown kind: ${docs.kinds.fetchRequest}`]);
const made = await askOnce({ fetchEnabled, kind: "gamecult.persona.chat_fetch_request.v9", tuple: fetchTuple, mouth });
assert.equal(made.document.schemaId, off.document.schemaId);
assert.equal(made.document.recordKey, off.document.recordKey);
assert.deepEqual(Buffer.from(made.document.payload), Buffer.from(off.document.payload), "byte for byte the answer to a kind that never existed");
}
});
test("fetch flag on: the request reaches the Persona's mouth and the receipt comes back typed, 5 slots; another Persona's request is UnknownPersona", async () => {
const { asked, mouth } = fetchStub();
const on = await askOnce({ fetchEnabled: true, kind: docs.kinds.fetchRequest, tuple: fetchTuple, mouth });
assert.equal(on.document.schemaId, docs.kinds.fetchReceipt);
assert.deepEqual(asked, [{ requestId: "req-f1", persona: "eureka", room: "#swarm:gamecult.org", eventId: "$file1" }]);
assert.deepEqual(docs.parseFetchReceipt(msgpack.decode(on.document.payload)), { requestId: "req-f1", outcome: "NoAttachment", roomId: null, image: null, receipt: null });
const ghost = await askOnce({ fetchEnabled: true, kind: docs.kinds.fetchRequest, tuple: docs.fetchRequestTuple({ requestId: "r", persona: "ghost", room: "#a:b", eventId: "$e" }), mouth });
assert.equal(docs.parseFetchReceipt(msgpack.decode(ghost.document.payload)).outcome, "UnknownPersona");
assert.equal(asked.length, 1);
});
// ---- persona-chat-v7: the fetch request and its receipts. Huginn copies the msgpack byte for byte (its persona-chat-v9).
const v7Dir = resolve(root, "tests", "fixtures", "persona-chat-v7");
const v7Image = { path: `${"cd".repeat(32)}/${"ab".repeat(32)}.png`, mime: "image/png", size: 2048, sha256: "ab".repeat(32) };
const v7Receipt = receipt => ({ tuple: docs.fetchReceiptTuple({ requestId: "req-f1", outcome: "Stored", roomId: "!abc:gamecult.org", image: v7Image, receipt }), parse: docs.parseFetchReceipt });
const v7Samples = {
"gamecult.persona.chat_fetch_request.v1": { tuple: fetchTuple, parse: docs.parseFetchRequest },
"gamecult.persona.chat_fetch_receipt.v1.stored-nil": v7Receipt(null),
"gamecult.persona.chat_fetch_receipt.v1.stored-posted": v7Receipt("Posted"),
"gamecult.persona.chat_fetch_receipt.v1.stored-existing": v7Receipt("Existing"),
"gamecult.persona.chat_fetch_receipt.v1.stored-unverified": { tuple: docs.fetchReceiptTuple({ requestId: "req-f1", outcome: "StoredUnverified", roomId: "!abc:gamecult.org", image: v7Image }), parse: docs.parseFetchReceipt },
"gamecult.persona.chat_fetch_receipt.v1.refused": { tuple: docs.fetchReceiptTuple({ requestId: "req-f1", outcome: "BridgedRoom" }), parse: docs.parseFetchReceipt },
};
for (const [name, { tuple, parse }] of Object.entries(v7Samples)) {
test(`golden fixture: ${name} is byte-stable (msgpack and hex) and parses back, and re-encodes byte-equal`, async () => {
const bytes = Buffer.from(msgpack.encode(tuple));
if (process.env.UPDATE_FIXTURES === "1") { await mkdir(v7Dir, { recursive: true }); await writeFile(resolve(v7Dir, `${name}.msgpack`), bytes); await writeFile(resolve(v7Dir, `${name}.hex`), hexOf(bytes)); }
assert.deepEqual(bytes, await readFile(resolve(v7Dir, `${name}.msgpack`)), "wire bytes changed: Huginn decodes these fixtures");
assert.equal(await readFile(resolve(v7Dir, `${name}.hex`), "utf8"), hexOf(bytes), "the hex dump is the msgpack file");
const decoded = msgpack.decode(bytes);
assert.deepEqual(parse(decoded), parse(tuple));
assert.equal(decoded.length, name.includes("request") ? 4 : 5);
if (!name.includes("request")) assert.deepEqual(Buffer.from(msgpack.encode(docs.fetchReceiptTuple(parse(decoded)))), bytes, "decode then re-encode is byte-equal");
});
}

View file

@ -22,6 +22,8 @@ export const kinds = Object.freeze({
refusal: "gamecult.persona.chat_refusal.v1",
openDirectRequest: "gamecult.persona.chat_open_direct_request.v1",
openDirectReceipt: "gamecult.persona.chat_open_direct_receipt.v1",
fetchRequest: "gamecult.persona.chat_fetch_request.v1", // served only when the mouth's CHAT_FETCH flag is on, else an unknown kind
fetchReceipt: "gamecult.persona.chat_fetch_receipt.v1",
});
export const refusals = Object.freeze(["Brake", "PersonaMismatch", "UnknownRoom", "EmptyText", "SendFailed", "UnknownPersona"]);
@ -35,6 +37,13 @@ const refusalDocument = Object.freeze([...refusals, "UnknownKind"]);
// One outcome word per open-direct receipt. room_id and event_id are present exactly for Opened and Existing.
export const directOutcomes = Object.freeze(["Opened", "Existing", "Brake", "PersonaMismatch", "UnknownPersona", "EmptyText", "ForeignUser", "NotAPerson", "NoSuchUser", "InvitePending", "Declined", "DailyLimit", "SendFailed"]);
// One outcome word per fetch receipt, closed: no mouth text travels. room_id and image are present exactly for Stored and StoredUnverified;
// the receipt slot is nil, Posted or Existing and is non-nil only for Stored.
export const fetchOutcomes = Object.freeze(["Stored", "StoredUnverified", "Brake", "PersonaMismatch", "UnknownPersona", "NotAMember", "BridgedRoom", "TooLarge", "TypeRefused", "NoAttachment", "HashMismatch", "RateLimited", "FetchFailed", "StoreFailed"]);
export const fileReceivedType = "org.gamecult.file_received"; // the Matrix event the mouth posts as the Mind after it stored her file
export const fetchReceiptModes = Object.freeze(["Posted", "Existing"]);
const maxFetchIdBytes = 255;
const maxTextBytes = 4000;
export const maxNewer = 50;
const maxContext = 20;
@ -172,6 +181,38 @@ export function parseOpenDirectReceipt(t) {
return { requestId: text(requestId, "request_id"), outcome, roomId: roomId === null ? null : text(roomId, "room_id"), eventId: eventId === null ? null : text(eventId, "event_id") };
}
// A Persona asks her mouth to keep one named Matrix file: room (an alias or a room id) and event id, nothing else. No mxc, URL, hash or path
// exists in the tuple; the mouth reads the event itself. The id shapes are the mouth's to refuse by word (NotAMember, NoAttachment); here only
// the bound that every id field has.
const fetchId = (v, what) => {
text(v, what);
if (Buffer.byteLength(v, "utf8") > maxFetchIdBytes) throw new Error(`${what} is over ${maxFetchIdBytes} bytes`);
return v;
};
export function fetchRequestTuple({ requestId, persona, room, eventId }) {
return [text(requestId, "request_id"), text(persona, "persona"), fetchId(room, "room"), fetchId(eventId, "event_id")];
}
export function parseFetchRequest(t) {
const [requestId, persona, room, eventId] = tuple(t, 4, "chat_fetch_request");
return { requestId: text(requestId, "request_id"), persona: text(persona, "persona"), room: fetchId(room, "room"), eventId: fetchId(eventId, "event_id") };
}
function checkFetchReceipt(outcome, roomId, image, receipt) {
if (!fetchOutcomes.includes(outcome)) throw new Error(`unknown fetch outcome ${String(outcome)}`);
const stored = outcome === "Stored" || outcome === "StoredUnverified";
if (stored !== (roomId !== null && image !== null) || (!stored && (roomId !== null || image !== null))) throw new Error("room_id and image are present exactly for Stored and StoredUnverified");
if (receipt !== null && !(outcome === "Stored" && fetchReceiptModes.includes(receipt))) throw new Error("receipt is nil, or Posted or Existing for Stored only");
}
export function fetchReceiptTuple({ requestId, outcome, roomId = null, image = null, receipt = null }) {
checkFetchReceipt(outcome, roomId, image, receipt);
return [text(requestId, "request_id"), outcome, roomId === null ? null : text(roomId, "room_id"), imageTuple(image), receipt];
}
export function parseFetchReceipt(t) {
const [requestId, outcome, roomId, image, receipt] = tuple(t, 5, "chat_fetch_receipt");
checkFetchReceipt(outcome, roomId, image, receipt);
return { requestId: text(requestId, "request_id"), outcome, roomId: roomId === null ? null : text(roomId, "room_id"), image: image === null ? null : parseImage(image), receipt };
}
export function deliveryTuple(d) {
return [
seq(d.seq, "seq"), text(d.roomAlias, "room_alias"), text(d.roomId, "room_id"), matrixId("$", d.eventId, "event_id"),
@ -230,6 +271,10 @@ export function parseDeliveryV4(t) {
// A Matrix id as the tuple carries it: its sigil, a non-empty rest, at most 255 bytes, no Unicode whitespace, control, format or private-use characters and no unpaired surrogates.
// The mouth applies the same test to what a homeserver returned, so a hostile id is no reply_to rather than a skipped delivery.
export const isMatrixId = (sigil, v) => typeof v === "string" && Buffer.byteLength(v) <= 255 && v.length > 1 && v[0] === sigil && !/[\p{White_Space}\p{Cc}\p{Cf}\p{Co}\p{Default_Ignorable_Code_Point}\uD800-\uDFFF]/u.test(v);
// A Matrix id the fetch path will put in a URL path or a request it builds: isMatrixId, and none of the characters or sequences that
// steer a path or a query (a slash, ?, #, %, backslash, ..). Stricter than isMatrixId on purpose: the event id and the album root are
// the sender's words. A room version 3 event id (standard base64) may hold a slash and is refused, as the fail-closed side.
export const isFetchId = (sigil, v) => isMatrixId(sigil, v) && !/[/?#%\\]/.test(v.slice(1)) && !v.includes("..");
function replyTo(r) {
if (r === null) return null;
const { eventId, sender, excerpt = null } = r;

View file

@ -87,10 +87,10 @@ export function classify(bytes) {
// GET the attachment named by `mxc` as this Persona: its token, its homeserver, the authenticated media path. The
// body is read stream-wise and abandoned the moment it passes the cap, whatever Content-Length said.
// Returns { bytes, mime, ext } or { refusal }.
export async function fetchImage({ fetch: httpFetch, homeserver, token, mxc, timeoutMs = FETCH_TIMEOUT_MS }) {
export async function fetchImage({ fetch: httpFetch, homeserver, token, mxc, timeoutMs = FETCH_TIMEOUT_MS, signal: outer }) {
const named = typeof mxc === "string" ? MXC.exec(mxc) : null;
if (!named) return { refusal: "no-url" };
const signal = AbortSignal.timeout(timeoutMs);
const signal = outer ? AbortSignal.any([AbortSignal.timeout(timeoutMs), outer]) : AbortSignal.timeout(timeoutMs); // `outer`: the caller's deadline, joined with this fetch's own
try {
const response = await httpFetch(`${homeserver}/_matrix/client/v1/media/download/${named[1]}/${named[2]}`, {
method: "GET", headers: { authorization: `Bearer ${token}` }, redirect: "error", signal,

View file

@ -64,6 +64,10 @@ const DM_SCAN = 200; // joined rooms read when looking for an existing DM with a
const NOT_E2EE = "This room is not end-to-end encrypted: the homeserver can read what is said here.";
const dayOf = ms => new Date(ms).toISOString().slice(0, 10);
const TAIL_TTL_MS = 15 * 1000;
const FETCH_DEADLINE_MS = 15000; // one fetch, whole: it ends before Huginn's 20 s wait
const FETCH_PER_HOUR = 30; // fetches one Persona may ask in an hour
const RELATION_PAGES = 3; // pages of a receipt target's relations read before the answer is called unproven
const PLATFORM = ["disc", "ord"].join(""); // the bridged platform's name, as Huginn's room_scope spells it: never written out whole
const HEAD_PAGE = 50; // the room head a gated send reads; also the most ids a RoomMoved receipt names
// The kind of an event for a delivery: an edit (m.replace) and an automated notice (m.notice) are not messages anyone is addressed by.
@ -121,7 +125,7 @@ export function createRoomGate() {
export function createMouth({
persona, homeserver, serverName, rooms, token, fetch: httpFetch = globalThis.fetch, state, brake,
now = Date.now, log = () => {}, syncTimeoutMs = 30000, maxPending = MAX_PENDING, gate = createRoomGate(), sendTimeoutMs = SEND_TIMEOUT_MS,
presenceGraceMs = PRESENCE_GRACE_MS,
presenceGraceMs = PRESENCE_GRACE_MS, fetchDeadlineMs = FETCH_DEADLINE_MS,
replyLink = false, // resolve each reply's quoted event into delivery.replyTo (the v4 answer; one more event read per reply to a non-Mind)
images = null, // this Persona's image store (createImageStore), or null: images stay the typed text line
isServed = async () => false, // (async) whether a Matrix user id is an account this process speaks for: a Mind is not a person to open a DM with
@ -152,6 +156,7 @@ export function createMouth({
const processed = new Set(); // event ids read since the position last advanced
const noticed = new Set(); // rooms this mouth created where the not-E2EE first line has gone out
const personTails = new Map(); // person's user id -> the open in flight: opens for one person run one at a time
const fetchTimes = []; // when this Persona's recent fetches began: display-only rate count, reset by a restart
const openedOn = new Map(); // person's user id -> the UTC day this process last created a room for them: one creation per person per day
async function matrix(method, path, body, prefix = API, signal) {
@ -747,6 +752,128 @@ export function createMouth({
}
}
// A top-level refusal of the event itself, ported from Huginn's room_scope refuse_event: a sender whose localpart (after leading
// underscores) starts with the platform's name, or has it after an underscore, or followed by `bot` or an underscore; or a content
// key naming it. Identifier rules, not language reading.
const platformEvent = event => {
const local = String(event?.sender ?? "").replace(/^@+/, "").split(":")[0].toLowerCase();
return local.replace(/^_+/, "").startsWith(PLATFORM) || local.includes(`_${PLATFORM}`) || local.includes(`${PLATFORM}bot`) || local.includes(`${PLATFORM}_`)
|| Object.keys(event?.content ?? {}).some(key => key.toLowerCase().includes(PLATFORM));
};
const IMAGE_WORD = { "no-url": "NoAttachment", encrypted: "TypeRefused", "wrong-type": "TypeRefused", "bad-header": "TypeRefused", "too-many-pixels": "TooLarge", "too-big": "TooLarge", "fetch-failed": "FetchFailed", "store-failed": "StoreFailed" };
// A Persona asks her mouth to keep one named Matrix file. The request is a room and an event id; the mouth reads the event, takes
// its content.url, fetches and checks the bytes, stores them, and posts the contract's inert receipt as her. One deadline owns the
// whole of it. The answer is one fixed outcome word and, for a stored file, the resolved room id and the store reference.
async function fetchAttachment({ requestId, persona: asked, room, eventId }) {
const deadline = AbortSignal.timeout(fetchDeadlineMs);
const seen = { room: "-", event: "-", size: "-", type: "-", sha256: "-" };
const idOr = (sigil, v) => docs.isFetchId(sigil, v) ? v : "-"; // a hostile id never reaches the log
let answered = false; // one log line per fetch: a gate step that finishes after the deadline answered is silent
const done = (outcome, { image = null, roomId = null, receipt = null, logged = "skipped", cause = null } = {}) => {
if (!answered) say(`fetch persona=${persona} room=${seen.room} event=${seen.event} outcome=${outcome} size=${seen.size} type=${seen.type} sha256=${seen.sha256} receipt=${logged}${cause === null ? "" : ` cause=${cause}`}`);
answered = true;
return { requestId, outcome, roomId, image, receipt };
};
if (asked !== persona) return done("PersonaMismatch");
if (!(await released())) return done("Brake");
if (!images) return done("StoreFailed");
const at = now();
while (fetchTimes.length > 0 && at - fetchTimes[0] >= 3_600_000) fetchTimes.shift();
if (fetchTimes.length >= FETCH_PER_HOUR) return done("RateLimited");
fetchTimes.push(at);
seen.event = idOr("$", eventId);
if (!docs.isFetchId("$", eventId)) return done("NoAttachment");
if (!docs.isFetchId("!", room) && !docs.isFetchId("#", room)) return done("NotAMember");
let stored = false; // images.put has run: a deadline after it leaves a file and no receipt
const aborted = new Promise((_, reject) => deadline.addEventListener("abort", () => reject(deadline.reason), { once: true }));
aborted.catch(() => {});
const within = promise => Promise.race([promise, aborted]);
try {
if (!started || unjoined()) await within(start());
const roomId = speakableRoomId(room) ?? room;
if (!docs.isFetchId("!", roomId) && !docs.isFetchId("#", roomId)) return done("NotAMember"); // an alias the directory answered with a strange id
seen.room = idOr("!", roomId);
return await within(gate.run(roomId, async () => {
deadline.throwIfAborted(); // re-checked after the gate wait: a queued fetch does not run past its deadline
if (!(await released())) return done("Brake"); // nor past a brake placed while it waited
const read = (method, path, body, prefix = API) => matrix(method, path, body, prefix, deadline);
const enc = encodeURIComponent;
const member = (await read("GET", "/joined_rooms")).joined_rooms;
if (!Array.isArray(member) || !member.includes(roomId)) return done("NotAMember");
const state = await read("GET", `/rooms/${enc(roomId)}/state`);
if (!Array.isArray(state)) return done("FetchFailed");
if (state.some(e => e?.type === "m.bridge" || e?.type === "uk.half-shot.bridge")) return done("BridgedRoom");
if (state.some(e => e?.type === "m.room.encryption")) return done("TypeRefused");
let event;
try { event = await read("GET", `/rooms/${enc(roomId)}/event/${enc(eventId)}`); } catch (error) {
if (error.status === 403 || error.status === 404) return done("NotAMember");
throw error;
}
if (event?.room_id !== roomId) return done("NotAMember");
if (!docs.isFetchId("$", event.event_id)) return done("NoAttachment"); // the receipt names the event the homeserver returned, not the one asked for
if (platformEvent(event)) return done("BridgedRoom");
const content = event.content;
if (event.type === "m.room.encrypted") return done("TypeRefused");
const relates = content?.["m.relates_to"];
if (event.type !== "m.room.message" || content === null || typeof content !== "object" || Object.keys(content).length === 0 || relates?.rel_type === "m.replace") return done("NoAttachment");
if (content.file !== undefined) return done("TypeRefused");
if (content.msgtype === "m.audio" || content.msgtype === "m.video") return done("TypeRefused");
if (content.msgtype !== "m.image" && content.msgtype !== "m.file") return done("NoAttachment");
if (typeof content.url !== "string" || !content.url.startsWith("mxc://")) return done("NoAttachment");
// The album root is the sender's word: it names a URL path below, so it must be an event id or nothing is built from it.
const albumRoot = relates?.rel_type === "m.thread" ? relates.event_id : null;
if (relates?.rel_type === "m.thread" && !docs.isFetchId("$", albumRoot)) return done("NoAttachment");
const got = await fetchImage({ fetch: httpFetch, homeserver, token, mxc: content.url, signal: deadline });
deadline.throwIfAborted();
if (got.refusal) return done(IMAGE_WORD[got.refusal] ?? "FetchFailed");
const computed = createHash("sha256").update(got.bytes).digest("hex");
Object.assign(seen, { size: got.bytes.length, type: got.mime, sha256: computed });
const info = content.info;
const claimed = info !== null && typeof info === "object" && !Array.isArray(info) && Object.hasOwn(info, "sha256");
if (claimed && info.sha256 !== computed) return done("HashMismatch");
deadline.throwIfAborted();
let image;
try { image = images.put(roomId, got.bytes, got.mime); } catch (error) { return done("StoreFailed", { cause: causeOf(error) }); }
stored = true;
if (!claimed) return done("StoredUnverified", { image, roomId });
const { receipt, word, cause } = await postFileReceipt(roomId, albumRoot, event.event_id, computed, got.bytes.length, deadline);
return done("Stored", { image, roomId, receipt, logged: word, cause });
}));
} catch (error) {
return done("FetchFailed", { logged: stored ? "failed" : "skipped", cause: causeOf(error) });
}
}
// The contract's receipt for a file the mouth stored, posted as her own user: a thread reply when the file is in an album thread,
// else a plain event referencing the file. Idempotence is Matrix's: her own earlier receipt in the target's relations. A relations
// read or a PUT that fails, a held brake, proves nothing and posts nothing: the file stays stored and the receipt is nil. A deadline
// is not such a failure and leaves the caller (it throws).
async function postFileReceipt(roomId, albumRoot, fileEventId, sha256, size, deadline) {
const none = (word, cause = null) => ({ receipt: null, word, cause });
try {
const room = encodeURIComponent(roomId);
const [target, relation] = albumRoot === null ? [fileEventId, "m.reference"] : [albumRoot, "m.thread"];
let from = null, exhausted = false;
for (let page = 0; page < RELATION_PAGES && !exhausted; page++) {
const reply = await matrix("GET", `/rooms/${room}/relations/${encodeURIComponent(target)}/${relation}/${docs.fileReceivedType}?dir=b&limit=100${from === null ? "" : `&from=${encodeURIComponent(from)}`}`, undefined, V1, deadline);
const mine = (reply?.chunk ?? []).some(e => e?.type === docs.fileReceivedType && e.sender === selfId && !e.unsigned?.redacted_because && e.content?.[docs.fileReceivedType]?.event_id === fileEventId);
if (mine) return { receipt: "Existing", word: "existing" };
if (typeof reply?.next_batch === "string" && reply.next_batch !== "") from = reply.next_batch; else exhausted = true;
}
if (!exhausted) return none("failed"); // three pages and more remain: not proven
if (!(await released())) return none("skipped"); // read again immediately before the PUT, as send does inside its gate
deadline.throwIfAborted();
const relates = albumRoot === null ? { rel_type: "m.reference", event_id: fileEventId } : { rel_type: "m.thread", event_id: albumRoot };
const txn = `fr-${createHash("sha256").update(`${persona}|${fileEventId}`).digest("hex")}`;
await matrix("PUT", `/rooms/${room}/send/${docs.fileReceivedType}/${txn}`, { "m.relates_to": relates, [docs.fileReceivedType]: { event_id: fileEventId, sha256, size } }, API, deadline);
return { receipt: "Posted", word: "posted" };
} catch (error) {
if (deadline.aborted) throw error;
return none("failed", causeOf(error));
}
}
async function run(signal) {
const pause = ms => new Promise(r => { const t = setTimeout(r, ms); signal?.addEventListener("abort", () => { clearTimeout(t); r(); }, { once: true }); });
while (!signal?.aborted) {
@ -760,7 +887,12 @@ export function createMouth({
}
}
return { start, syncOnce, ack, read, pendingAfter, tail, speakableRooms, send, openDirect, run, get userId() { return selfId; }, get pendingCount() { return pending.length; } };
return { start, syncOnce, ack, read, pendingAfter, tail, speakableRooms, send, openDirect, fetchAttachment, run, get userId() { return selfId; }, get pendingCount() { return pending.length; } };
}
// The log's word for a failure: error class and HTTP status only, never the message (it can hold a URL, an mxc or a sender-chosen id).
function causeOf(error) {
return `${/^[A-Za-z]{1,32}$/.test(error?.name) ? error.name : "Error"}/${Number.isInteger(error?.status) ? error.status : "-"}`;
}
// Replace every secret in a message. `secrets` is one string or a list.
@ -845,7 +977,7 @@ export function pageOf(persona, deliveries, msgpack, log = () => {}, v3 = false,
// session, and a request is answered on the session it arrived on. Requests arrive as
// document_put_raw messages. A request naming a Persona this
// process does not serve is answered with a typed UnknownPersona refusal.
export async function startChatEndpoint({ mouths, runtime, port, connectionKey, runtimeId, socketFactory = createSocket, host = "127.0.0.1", log = () => {}, replyLink = false }) {
export async function startChatEndpoint({ mouths, runtime, port, connectionKey, runtimeId, socketFactory = createSocket, host = "127.0.0.1", log = () => {}, replyLink = false, fetchEnabled = false }) {
if (!/^127\.\d+\.\d+\.\d+$/.test(host)) throw new Error("the chat endpoint listens on loopback only");
const { msgpack, cultnet } = runtime;
const socket = socketFactory("udp4");
@ -857,7 +989,7 @@ export async function startChatEndpoint({ mouths, runtime, port, connectionKey,
const respond = (peer, persona, kind, recordKey, tuple) => server.send(peer, "schema", msgpack.encode(cultnet.encodeCultNetMessageForWire(
docs.envelope({ kind, recordKey, payload: msgpack.encode(tuple), sourceRuntimeId: runtimeId, sourceAgentId: persona, sourceRole: "bifrost-persona-matrix-mouth" }), "cultnet.schema.v0")));
const refuseUnknown = (peer, persona) => respond(peer, persona, docs.kinds.refusal, persona, docs.refusalTuple({ persona, refusal: "UnknownPersona" }));
const served = [docs.kinds.pendingRequest, docs.kinds.pendingRequestV3, ...(replyLink ? [docs.kinds.pendingRequestV4] : []), docs.kinds.ack, docs.kinds.read, docs.kinds.sendRequest, docs.kinds.sendRequestV2, docs.kinds.openDirectRequest];
const served = [docs.kinds.pendingRequest, docs.kinds.pendingRequestV3, ...(replyLink ? [docs.kinds.pendingRequestV4] : []), docs.kinds.ack, docs.kinds.read, docs.kinds.sendRequest, docs.kinds.sendRequestV2, docs.kinds.openDirectRequest, ...(fetchEnabled ? [docs.kinds.fetchRequest] : [])];
const handle = async ({ peer, channelId, payload }) => {
try {
if (channelId !== "schema") throw new Error("chat request arrived on a foreign channel");
@ -874,6 +1006,11 @@ export async function startChatEndpoint({ mouths, runtime, port, connectionKey,
const mouth = mouths.get(request.persona);
const receipt = mouth ? await mouth.openDirect(request) : { requestId: request.requestId, outcome: "UnknownPersona", roomId: null, eventId: null };
respond(peer, request.persona, docs.kinds.openDirectReceipt, receipt.requestId, docs.openDirectReceiptTuple(receipt));
} else if (document.schemaId === docs.kinds.fetchRequest) { // only reachable with fetchEnabled: `served` drops it otherwise
const request = docs.parseFetchRequest(tuple);
const mouth = mouths.get(request.persona);
const receipt = mouth ? await mouth.fetchAttachment(request) : { requestId: request.requestId, outcome: "UnknownPersona", roomId: null, image: null, receipt: null };
respond(peer, request.persona, docs.kinds.fetchReceipt, receipt.requestId, docs.fetchReceiptTuple(receipt));
} else if (document.schemaId === docs.kinds.pendingRequest || document.schemaId === docs.kinds.pendingRequestV3 || document.schemaId === docs.kinds.pendingRequestV4) {
const v4 = document.schemaId === docs.kinds.pendingRequestV4; // only reachable with replyLink: `served` drops it otherwise
const v3 = v4 || document.schemaId === docs.kinds.pendingRequestV3; // the kind asked by names the answer: v1 gets the 12-field v2 answer, as before
@ -935,6 +1072,8 @@ export function loadRuntime(cultRoot) {
// The v4 flag: on only for exactly "1"; unset, "0", "true", "yes" and anything else are off.
export const replyLinkFromEnv = env => env.CHAT_PENDING_V4 === "1";
// The fetch flag, the same way: on only for exactly "1". Off, the fetch request kind is an unknown kind.
export const fetchFromEnv = env => env.CHAT_FETCH === "1";
function required(env, name) {
if (!env[name]) throw new Error(`${name} is required`);
@ -983,6 +1122,7 @@ export async function main(env = process.env) {
const controlRoot = env.CONTROL_ROOT ?? "/etc/gamecult/minds";
const imageRoot = env.IMAGE_STORE_ROOT ? resolve(env.IMAGE_STORE_ROOT) : null; // unset: chat images stay the typed text line
const gate = createRoomGate(); // one for every Persona: two Minds in one room take turns
const fetchEnabled = fetchFromEnv(env);
const replyLink = replyLinkFromEnv(env); // default off: Huginn pins the v3 tuple bytes until it reads v4
const mouths = new Map(personas.map(persona => {
mkdirSync(join(stateRoot, persona), { recursive: true });
@ -995,7 +1135,7 @@ export async function main(env = process.env) {
}));
const listen = required(env, "CHAT_RUDP_LISTEN"); // host:port, loopback
const colon = listen.lastIndexOf(":");
await startChatEndpoint({ mouths, runtime, host: listen.slice(0, colon), port: Number(listen.slice(colon + 1)), connectionKey: chatKey, runtimeId: "bifrost-persona-mouth", log, replyLink });
await startChatEndpoint({ mouths, runtime, host: listen.slice(0, colon), port: Number(listen.slice(colon + 1)), connectionKey: chatKey, runtimeId: "bifrost-persona-mouth", log, replyLink, fetchEnabled });
await Promise.all([...mouths.values()].map(mouth => mouth.run()));
});
}