538 lines
31 KiB
JavaScript
538 lines
31 KiB
JavaScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { createHash } from "node:crypto";
|
|
import { createRequire } from "node:module";
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdtemp, mkdir, readFile, readdir, stat, writeFile, copyFile } from "node:fs/promises";
|
|
import { existsSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { resolve } from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
|
// The fixture is the store Idunn's incident cut wrote: GameCult/Idunn branch
|
|
// hands/watchdog-idunn-incident, commit 7f7528b33e8e5c607c19cc7f8f8a8da4def50fca
|
|
// ("Add the idunn.operator_incident.v1 wire fixture"), tests/fixtures/ there.
|
|
// It holds two records: closed-target (closed, recovered) and open-target.
|
|
const FIXTURE = resolve(import.meta.dirname, "fixtures", "idunn.operator_incident.v1.cc");
|
|
const FIXTURE_SHA256 = "f446770b07da99204e9036ef521f3136be6251e5b0a68e4e9533a89f25f18cab";
|
|
const CLOSED_KEY = "continuity-exhausted:closed-target:1700000000000";
|
|
const OPEN_KEY = "continuity-exhausted:open-target:1700000100000";
|
|
const TOOL = resolve(import.meta.dirname, "..", "tools", "watchdog-notice.mjs");
|
|
const BRIDGE = resolve(import.meta.dirname, "..", "tools", "bifrost-bridge.mjs");
|
|
const RECIPIENT = "424242RECIPIENTCANARY";
|
|
|
|
const OPENED_TEXT = (subject, at) => `Idunn incident opened: continuity-exhausted on \`${subject}\` at ${at}`;
|
|
const T_OPEN = "2023-11-14T22:13:20.000Z";
|
|
const T_CLOSE = "2023-11-14T22:15:00.000Z";
|
|
const CLOSED_TEXT = `Idunn incident closed (recovered): continuity-exhausted on \`closed-target\`, opened ${T_OPEN}, closed ${T_CLOSE}`;
|
|
|
|
// CultLib is the sibling ../CultLib, like the other tests.
|
|
const cult = resolve(import.meta.dirname, "..", "..", "CultLib");
|
|
const cache = createRequire(resolve(cult, "packages", "cultcache-ts", "package.json"))("@gamecult/cultcache-ts");
|
|
const msgpack = createRequire(resolve(cult, "packages", "cultnet-ts", "package.json"))("@msgpack/msgpack");
|
|
|
|
// A bridge double that logs each call (with the content it was handed) and
|
|
// behaves as the `mode` file says: ok; fail (exit 75, retryable); die
|
|
// (exit 1 after "sending"); signal (killed after "sending"); hang (never answers after
|
|
// "sending"); flood (answers with more stdout than spawnSync buffers); or crash (kills its
|
|
// parent, the reader, after logging, as a power cut between send and journal would).
|
|
const FAKE_BRIDGE = `
|
|
import { appendFileSync, readFileSync } from "node:fs";
|
|
import { dirname, resolve } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
const args = process.argv.slice(2);
|
|
const opt = (name) => args[args.indexOf("--" + name) + 1];
|
|
const calls = resolve(here, "calls.jsonl");
|
|
const n = (() => { try { return readFileSync(calls, "utf8").trim().split("\\n").length; } catch { return 0; } })();
|
|
appendFileSync(calls, JSON.stringify({ args, content: readFileSync(opt("content-file"), "utf8") }) + "\\n");
|
|
const mode = readFileSync(resolve(here, "mode"), "utf8").trim();
|
|
if (mode === "crash") { process.kill(process.ppid, "SIGKILL"); process.exit(0); }
|
|
if (mode === "fail" || mode === "die") { process.stderr.write("boom " + opt("recipient-id")); process.stdout.write("boom " + opt("recipient-id")); process.exit(mode === "fail" ? 75 : 1); }
|
|
if (mode === "signal") process.kill(process.pid, "SIGKILL");
|
|
if (mode === "hang") setTimeout(() => {}, 30000);
|
|
else if (mode === "flood") process.stdout.write("x".repeat(2 * 1024 * 1024));
|
|
else
|
|
process.stdout.write(JSON.stringify({ action: "discord-dm", ok: true, messageId: "m" + (n + 1) }));
|
|
`;
|
|
|
|
async function world(prefix, { incidents = FIXTURE, mode = "ok" } = {}) {
|
|
const dir = await mkdtemp(resolve(tmpdir(), prefix));
|
|
const w = {
|
|
dir,
|
|
incidents: resolve(dir, "idunn", "incidents.cc"),
|
|
journal: resolve(dir, "bifrost", "journal.cc"),
|
|
receipts: resolve(dir, "bifrost", "receipts.cc"),
|
|
bridge: resolve(dir, "fake-bridge.mjs"),
|
|
calls: resolve(dir, "calls.jsonl"),
|
|
};
|
|
await mkdir(resolve(dir, "idunn"));
|
|
await writeFile(w.bridge, FAKE_BRIDGE);
|
|
await writeFile(resolve(dir, "mode"), mode);
|
|
if (incidents) await copyFile(incidents, w.incidents);
|
|
w.setMode = (value) => writeFile(resolve(dir, "mode"), value);
|
|
w.run = (extra = {}) => run(w, extra);
|
|
w.status = (env) => runStatus(w, env);
|
|
w.posts = async () => (existsSync(w.calls) ? (await readFile(w.calls, "utf8")).trim().split("\n").map((line) => JSON.parse(line)) : []);
|
|
return w;
|
|
}
|
|
|
|
// The reader reads Date.now(). A test drives the backoff the way it stubs fetch:
|
|
// a preload (fake-clock.mjs, passed to the reader as --import) fixes Date.now
|
|
// to the minute (from T0) the run happens at. The reader has no clock seam.
|
|
const T0 = 1_800_000_000_000;
|
|
const CLOCK = resolve(import.meta.dirname, "fake-clock.mjs");
|
|
const at = (minutes) => ({ FAKE_NOW_MS: String(T0 + Math.round(minutes * 60_000)) });
|
|
|
|
function run(w, { env = {}, bridge = w.bridge } = {}) {
|
|
return spawnSync(process.execPath, ["--import", CLOCK, TOOL, "process", "--incident-store", w.incidents, "--journal-store", w.journal, "--receipt-store", w.receipts, "--bridge-cli", bridge], {
|
|
encoding: "utf8",
|
|
env: { ...process.env, DISCORD_OWNER_ID: RECIPIENT, ...env },
|
|
});
|
|
}
|
|
|
|
function runStatus(w, env = {}) {
|
|
const result = spawnSync(process.execPath, ["--import", CLOCK, TOOL, "status", "--journal-store", w.journal], { encoding: "utf8", env: { ...process.env, ...env } });
|
|
return { ...result, rows: result.status === null || !result.stdout ? [] : JSON.parse(result.stdout) };
|
|
}
|
|
|
|
const opt = (post, name) => post.args[post.args.indexOf(`--${name}`) + 1];
|
|
const notice = (post) => opt(post, "cultmesh-command-id").split("#")[1];
|
|
|
|
// Builds an incident store from the fixture's own envelopes with the payload
|
|
// tuples rewritten by `edit`.
|
|
async function variantStore(path, edit) {
|
|
const source = await new cache.SingleFileMessagePackBackingStore(FIXTURE).pullAll();
|
|
const tuples = edit(source.map((envelope) => msgpack.decode(envelope.payload)));
|
|
const envelopes = tuples.map((tuple, index) => ({ ...source[0], key: `variant-${index}-${tuple[1]}`, payload: msgpack.encode(tuple), schemaId: source[0].schemaId, catalogEntry: source[0].catalogEntry }));
|
|
await new cache.SingleFileMessagePackBackingStore(path).pushAll(envelopes);
|
|
}
|
|
|
|
test("the Idunn fixture decodes as the incidents Idunn wrote", async () => {
|
|
const bytes = await readFile(FIXTURE);
|
|
assert.equal(createHash("sha256").update(bytes).digest("hex"), FIXTURE_SHA256);
|
|
const w = await world("wn-fixture-");
|
|
const result = w.run();
|
|
assert.equal(result.status, 0, result.stderr);
|
|
const posts = await w.posts();
|
|
const byKey = Object.fromEntries(posts.map((post) => [opt(post, "cultmesh-command-id"), post.content]));
|
|
assert.deepEqual(byKey, {
|
|
[`${CLOSED_KEY}#opened`]: OPENED_TEXT("closed-target", T_OPEN),
|
|
[`${CLOSED_KEY}#closed`]: CLOSED_TEXT,
|
|
[`${OPEN_KEY}#opened`]: OPENED_TEXT("open-target", T_CLOSE),
|
|
});
|
|
});
|
|
|
|
test("one incident yields one opening post across runs", async () => {
|
|
const w = await world("wn-once-");
|
|
assert.equal(w.run().status, 0);
|
|
assert.equal(w.run().status, 0);
|
|
const posts = await w.posts();
|
|
const openings = posts.filter((post) => notice(post) === "opened");
|
|
assert.equal(openings.length, 2);
|
|
assert.deepEqual(openings.map((post) => opt(post, "source-id")).sort(), [CLOSED_KEY, OPEN_KEY]);
|
|
assert.equal(posts.length, 3, "no second run posts again");
|
|
for (const post of posts) {
|
|
const key = opt(post, "cultmesh-command-id");
|
|
assert.equal(opt(post, "recipient-id"), RECIPIENT);
|
|
assert.equal(opt(post, "nonce"), createHash("sha256").update(key).digest("hex").slice(0, 25));
|
|
assert.equal(opt(post, "receipt-store"), w.receipts);
|
|
assert.equal(opt(post, "source-kind"), "idunn-operator-incident");
|
|
}
|
|
});
|
|
|
|
test("closure is posted once and only after a completed opening", async () => {
|
|
const w = await world("wn-closure-");
|
|
assert.equal(w.run().status, 0);
|
|
assert.equal(w.run().status, 0);
|
|
const closures = (await w.posts()).filter((post) => notice(post) === "closed");
|
|
assert.equal(closures.length, 1);
|
|
assert.equal(opt(closures[0], "source-id"), CLOSED_KEY);
|
|
|
|
// An opening that never completes blocks its closure, now and later.
|
|
const failing = await world("wn-closure-failed-", { mode: "fail" });
|
|
for (const minute of [0, 1, 3, 7, 15]) failing.run({ env: at(minute) });
|
|
assert.equal((await failing.posts()).filter((post) => notice(post) === "closed").length, 0);
|
|
const rows = (await failing.status()).rows;
|
|
assert.deepEqual(rows.map((row) => `${row.notice}:${row.status}:${row.attempts}`), ["opened:failed:5", "opened:failed:5"]);
|
|
await failing.setMode("ok");
|
|
assert.equal(failing.run({ env: at(31) }).status, 0);
|
|
assert.equal(failing.run({ env: at(32) }).status, 0);
|
|
assert.deepEqual((await failing.posts()).filter((post) => notice(post) === "closed").map((post) => opt(post, "source-id")), [CLOSED_KEY], "the closure follows the completed opening, once");
|
|
|
|
const crashed = await world("wn-closure-unknown-", { mode: "crash" });
|
|
assert.equal(crashed.run().status, null, "the reader died mid-send");
|
|
await crashed.setMode("ok");
|
|
crashed.run();
|
|
crashed.run();
|
|
assert.equal((await crashed.posts()).filter((post) => notice(post) === "closed").length, 0);
|
|
});
|
|
|
|
test("a running entry becomes unknown and is never re-posted", async () => {
|
|
const w = await world("wn-running-", { mode: "crash" });
|
|
assert.equal(w.run().status, null);
|
|
assert.equal((await w.status()).rows[0].status, "running");
|
|
await w.setMode("ok");
|
|
const recovered = w.run();
|
|
assert.notEqual(recovered.status, 0, "an unknown entry is a delivery failure");
|
|
assert.equal(JSON.parse(recovered.stdout).unknown, 1);
|
|
const posts = await w.posts();
|
|
assert.equal(posts.filter((post) => opt(post, "cultmesh-command-id") === `${CLOSED_KEY}#opened`).length, 1, "the interrupted notice is not sent again");
|
|
const rows = (await w.status()).rows;
|
|
assert.equal(rows.find((row) => row.incidentKey === CLOSED_KEY && row.notice === "opened").status, "unknown");
|
|
assert.notEqual(w.run({ env: at(100_000) }).status, 0, "still failing, and not re-posted, long past any backoff");
|
|
assert.equal((await w.posts()).length, posts.length);
|
|
});
|
|
|
|
test("an entry stamped ahead of the clock is eligible now, not deferred by the skew", async () => {
|
|
const w = await world("wn-skew-", { mode: "fail" });
|
|
w.run({ env: at(1000) });
|
|
assert.equal((await w.posts()).length, 2, "both openings tried once, stamped at minute 1000");
|
|
w.run({ env: at(0) });
|
|
assert.equal((await w.posts()).length, 4, "the clock is behind the stamp: retried at once");
|
|
assert.equal((await w.status(at(0))).rows[0].nextEligibleAt, new Date(T0 + 2 * 60_000).toISOString(), "the retry rewrote the stamp from the corrected clock");
|
|
w.run({ env: at(1.5) });
|
|
assert.equal((await w.posts()).length, 4, "the cadence holds from the rewritten stamp");
|
|
w.run({ env: at(2) });
|
|
assert.equal((await w.posts()).length, 6);
|
|
});
|
|
|
|
test("the reader takes time from Date.now() alone, never from the environment", async () => {
|
|
const w = await world("wn-noseam-", { mode: "fail" });
|
|
w.run({ env: { ...at(0), WATCHDOG_NOTICE_NOW_MS: "-5" } });
|
|
w.run({ env: { ...at(0), WATCHDOG_NOTICE_NOW_MS: "8000000000000000" } });
|
|
assert.equal((await w.posts()).length, 2, "no environment variable moves the clock");
|
|
assert.equal((await w.status({ WATCHDOG_NOTICE_NOW_MS: "8000000000000000", ...at(0) })).rows[0].nextEligibleAt, new Date(T0 + 60_000).toISOString());
|
|
});
|
|
|
|
const backoffMinutes = [0, 1, 3, 7, 15, 31, 63, 123, 183];
|
|
|
|
test("a failed notice is retried after its backoff and never exhausted", async () => {
|
|
const w = await world("wn-retry-", { mode: "fail" });
|
|
// Run at every expected spawn minute and one minute before it: only the former may spawn.
|
|
const probes = [...new Set(backoffMinutes.flatMap((minute) => [minute - 1, minute]).filter((minute) => minute >= 0))].sort((a, b) => a - b);
|
|
const spawned = [];
|
|
for (const minute of probes) {
|
|
const before = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY).length;
|
|
w.run({ env: at(minute) });
|
|
const after = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY).length;
|
|
if (after > before) spawned.push(minute);
|
|
}
|
|
assert.deepEqual(spawned, backoffMinutes, "attempts at 1, 2, 4, 8, 16, 32 then every 60 minutes after the last attempt");
|
|
const open = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY);
|
|
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1, "every attempt carries the same nonce");
|
|
const rows = (await w.status()).rows;
|
|
assert.deepEqual(rows.map((row) => `${row.status}:${row.attempts}:${row.lastError}`), ["failed:9:bridge-not-sent", "failed:9:bridge-not-sent"], "never a terminal state");
|
|
const final = w.run({ env: at(183) });
|
|
const status = await w.status();
|
|
assert.notEqual(final.status, 0);
|
|
assert.notEqual(status.status, 0);
|
|
for (const text of [final.stdout, final.stderr, status.stdout, await readFile(w.journal, "latin1")]) {
|
|
assert.ok(!text.includes("RECIPIENTCANARY"), "bridge output and recipient never reach the reader's output or journal");
|
|
}
|
|
});
|
|
|
|
test("a failed notice is not retried before its delay", async () => {
|
|
const w = await world("wn-delay-", { mode: "fail" });
|
|
const count = async () => (await w.posts()).length;
|
|
w.run({ env: at(0) });
|
|
assert.equal(await count(), 2, "both openings tried once");
|
|
w.run({ env: at(0.99) });
|
|
w.run({ env: at(0) });
|
|
assert.equal(await count(), 2, "a run inside the delay spawns nothing");
|
|
const row = (await w.status(at(0))).rows[0];
|
|
assert.equal(row.attempts, 1);
|
|
assert.equal(row.nextEligibleAt, new Date(T0 + 60_000).toISOString(), "status names when the entry is next eligible");
|
|
w.run({ env: at(1) });
|
|
assert.equal(await count(), 4, "eligible once the delay has passed");
|
|
assert.equal((await w.status(at(1))).rows[0].nextEligibleAt, new Date(T0 + 3 * 60_000).toISOString());
|
|
});
|
|
|
|
test("the unit signals from the third failed attempt while still retrying", async () => {
|
|
const w = await world("wn-signal-", { mode: "fail" });
|
|
const exits = [];
|
|
for (const minute of [0, 1, 3]) exits.push(w.run({ env: at(minute) }).status);
|
|
assert.deepEqual(exits, [0, 0, 1], "zero after attempts 1 and 2, non-zero from 3");
|
|
assert.equal((await w.status()).status, 1);
|
|
const before = (await w.posts()).length;
|
|
assert.equal(w.run({ env: at(7) }).status, 1);
|
|
assert.equal((await w.posts()).length, before + 2, "attempt 4 still spawns");
|
|
assert.ok((await w.status()).rows.every((row) => row.status === "failed" && row.attempts === 4));
|
|
});
|
|
|
|
test("a notice that finally sends after many failures posts once", async () => {
|
|
const w = await world("wn-finally-", { mode: "fail" });
|
|
for (const minute of [0, 1, 3, 7, 15]) w.run({ env: at(minute) });
|
|
await w.setMode("ok");
|
|
assert.equal(w.run({ env: at(31) }).status, 0);
|
|
const open = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY);
|
|
assert.equal(open.length, 6, "five failures and one success");
|
|
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1, "the success carries the same nonce");
|
|
const before = (await w.posts()).length;
|
|
for (const minute of [32, 100, 1000]) assert.equal(w.run({ env: at(minute) }).status, 0);
|
|
assert.equal((await w.posts()).length, before, "completed is never posted again");
|
|
assert.deepEqual((await w.status()).rows.map((row) => `${row.status}:${row.attempts}`), ["completed:1", "completed:6", "completed:6"]);
|
|
});
|
|
|
|
test("the reader never writes or locks the incident store", async () => {
|
|
const w = await world("wn-readonly-");
|
|
const before = { bytes: await readFile(w.incidents), mtime: (await stat(w.incidents)).mtimeMs };
|
|
assert.equal(w.run().status, 0);
|
|
assert.equal(w.run().status, 0);
|
|
assert.deepEqual(await readFile(w.incidents), before.bytes);
|
|
assert.equal((await stat(w.incidents)).mtimeMs, before.mtime);
|
|
assert.deepEqual(await readdir(resolve(w.dir, "idunn")), ["incidents.cc"], "no lock or temp file beside the store");
|
|
});
|
|
|
|
test("notice content carries no sensitive text", async () => {
|
|
const w = await world("wn-egress-", { incidents: null });
|
|
await variantStore(w.incidents, (tuples) => {
|
|
const [closed, open] = tuples;
|
|
const longer = [...open, "extra-CANARYFIELD"];
|
|
const pathSubject = ["idunn.operator_incident.v1", `continuity-exhausted:/etc/CANARYPATH x:${open[4]}`, "continuity-exhausted", "/etc/CANARYPATH x", open[4], null, null];
|
|
const otherSchema = ["idunn.operator_incident.v0", "CANARYSCHEMA", "continuity-exhausted", "x", 1, null, null];
|
|
return [closed, longer, pathSubject, otherSchema];
|
|
});
|
|
const result = w.run();
|
|
assert.notEqual(result.status, 0, "refused records fail the run");
|
|
assert.deepEqual(JSON.parse(result.stdout), { skipped: 1, refused: 2, unknown: 0, failing: 0 });
|
|
const posts = await w.posts();
|
|
assert.deepEqual(posts.map((post) => opt(post, "source-id")), [CLOSED_KEY, CLOSED_KEY]);
|
|
for (const post of posts) assert.match(post.content, /^Idunn incident (opened: continuity-exhausted on `closed-target` at \d{4}-[\d-]+T[\d:.]+Z|closed \(recovered\): continuity-exhausted on `closed-target`, opened [\dTZ:.-]+, closed [\dTZ:.-]+)$/);
|
|
for (const text of [result.stdout, result.stderr, (await w.status()).stdout, await readFile(w.journal, "latin1"), JSON.stringify(posts)]) {
|
|
for (const canary of ["CANARYFIELD", "CANARYPATH", "CANARYSCHEMA"]) assert.ok(!text.includes(canary), "no byte of a refused record is echoed");
|
|
}
|
|
});
|
|
|
|
test("no recipient means no journal write and a non-zero exit", async () => {
|
|
for (const value of [undefined, "", " "]) {
|
|
const w = await world("wn-recipient-");
|
|
const env = value === undefined ? { DISCORD_OWNER_ID: undefined } : { DISCORD_OWNER_ID: value };
|
|
const result = spawnSync(process.execPath, [TOOL, "process", "--incident-store", w.incidents, "--journal-store", w.journal, "--receipt-store", w.receipts, "--bridge-cli", w.bridge], { encoding: "utf8", env: { ...process.env, ...env } });
|
|
assert.notEqual(result.status, 0);
|
|
assert.ok(!existsSync(w.journal));
|
|
assert.deepEqual(await w.posts(), []);
|
|
}
|
|
});
|
|
|
|
test("an absent incident store is a clean no-op", async () => {
|
|
const w = await world("wn-absent-", { incidents: null });
|
|
const result = w.run();
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.ok(!existsSync(w.journal));
|
|
assert.deepEqual(await w.posts(), []);
|
|
});
|
|
|
|
test("an emptied or restored incident store never re-sends", async () => {
|
|
const w = await world("wn-restore-");
|
|
assert.equal(w.run().status, 0);
|
|
const keys = async () => (await w.status()).rows.map((row) => `${row.incidentKey}#${row.notice}`).sort();
|
|
const all = [`${CLOSED_KEY}#closed`, `${CLOSED_KEY}#opened`, `${OPEN_KEY}#opened`];
|
|
assert.deepEqual(await keys(), all);
|
|
await variantStore(w.incidents, (tuples) => [tuples[1]]);
|
|
w.run();
|
|
await variantStore(w.incidents, () => []);
|
|
w.run();
|
|
await writeFile(w.incidents, "");
|
|
w.run();
|
|
assert.deepEqual(await keys(), all, "absence from Idunn's store deletes nothing");
|
|
await copyFile(FIXTURE, w.incidents);
|
|
assert.equal(w.run().status, 0);
|
|
assert.equal((await w.posts()).length, 3, "exactly one post per notice across all views of the store");
|
|
});
|
|
|
|
test("a bridge that dies after sending is unknown and never re-posted", async () => {
|
|
for (const mode of ["die", "signal"]) {
|
|
const w = await world(`wn-${mode}-`, { mode });
|
|
const first = w.run();
|
|
assert.notEqual(first.status, 0);
|
|
assert.equal(JSON.parse(first.stdout).unknown, 2);
|
|
assert.notEqual((await w.status()).status, 0, "status exits non-zero while an entry is unknown");
|
|
await w.setMode("ok");
|
|
assert.notEqual(w.run({ env: at(1000) }).status, 0, "far past any backoff");
|
|
assert.notEqual(w.run({ env: at(100_000) }).status, 0);
|
|
const posts = await w.posts();
|
|
assert.equal(posts.length, 2, "one opening post per incident, no retry, no closure");
|
|
assert.ok(posts.every((post) => notice(post) === "opened"));
|
|
assert.deepEqual((await w.status()).rows.map((row) => `${row.notice}:${row.status}:${row.attempts}:${row.lastError}`), ["opened:unknown:1:bridge-exit-unknown", "opened:unknown:1:bridge-exit-unknown"]);
|
|
assert.ok(!(await readFile(w.journal, "latin1")).includes("RECIPIENTCANARY"));
|
|
}
|
|
});
|
|
|
|
test("a spawn timeout and a spawn error are unknown, never failed, never re-posted", async () => {
|
|
for (const mode of ["hang", "flood"]) {
|
|
const w = await world(`wn-${mode}-`, { mode });
|
|
const first = w.run({ env: { WATCHDOG_NOTICE_BRIDGE_TIMEOUT_MS: "1500" } });
|
|
assert.notEqual(first.status, 0, mode);
|
|
assert.equal(JSON.parse(first.stdout).unknown, 2, mode);
|
|
await w.setMode("ok");
|
|
assert.notEqual(w.run({ env: at(1000) }).status, 0, "far past any backoff");
|
|
assert.notEqual(w.run({ env: at(100_000) }).status, 0);
|
|
const posts = await w.posts();
|
|
assert.equal(posts.length, 2, `${mode}: one opening post per incident, no retry, no closure`);
|
|
assert.ok(posts.every((post) => notice(post) === "opened"));
|
|
assert.deepEqual((await w.status()).rows.map((row) => `${row.notice}:${row.status}:${row.attempts}:${row.lastError}`), ["opened:unknown:1:bridge-exit-unknown", "opened:unknown:1:bridge-exit-unknown"], mode);
|
|
}
|
|
});
|
|
|
|
test("errors name a code and never echo a path or an input", async () => {
|
|
const w = await world("wn-canary-", { incidents: null });
|
|
await mkdir(resolve(w.dir, "CANARYDIR"));
|
|
const corrupt = resolve(w.dir, "CANARYDIR", "incidents.cc");
|
|
await writeFile(corrupt, "not a cultcache store CANARYBYTES");
|
|
const result = spawnSync(process.execPath, [TOOL, "process", "--incident-store", corrupt, "--journal-store", resolve(w.dir, "CANARYDIR", "j.cc"), "--receipt-store", w.receipts, "--bridge-cli", w.bridge], { encoding: "utf8", env: { ...process.env, DISCORD_OWNER_ID: RECIPIENT } });
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(result.stderr, /incident-store-unreadable/);
|
|
assert.ok(!(result.stdout + result.stderr).includes("CANARY"));
|
|
});
|
|
|
|
// The real bridge, reached through the real reader, with Discord's HTTP API
|
|
// stubbed at fetch (NODE_OPTIONS reaches the spawned bridge).
|
|
async function fetchStub(dir) {
|
|
const log = resolve(dir, "fetch.jsonl");
|
|
const stub = resolve(dir, "fetch-stub.mjs");
|
|
await writeFile(stub, `
|
|
import { appendFileSync } from "node:fs";
|
|
globalThis.fetch = async (url, init) => {
|
|
appendFileSync(${JSON.stringify(log)}, JSON.stringify({ url: String(url), body: JSON.parse(init.body) }) + "\\n");
|
|
const channels = String(url).endsWith("/users/@me/channels");
|
|
const behavior = process.env.STUB_BEHAVIOR;
|
|
if (channels && behavior === "dm-channel-500") return new Response("{}", { status: 500 });
|
|
const rejected = /^message-(\\d{3})$/.exec(behavior ?? "");
|
|
if (!channels && rejected) return new Response("{}", { status: Number(rejected[1]) });
|
|
if (!channels && behavior === "message-network-error") throw new TypeError("fetch failed");
|
|
return new Response(JSON.stringify(channels ? { id: "dm-channel" } : { id: "message-1" }), { status: 200 });
|
|
};
|
|
`);
|
|
return { log, options: `--import=${pathToFileURL(stub).href}` };
|
|
}
|
|
const fetches = async (log) => (existsSync(log) ? (await readFile(log, "utf8")).trim().split("\n").map((line) => JSON.parse(line)) : []);
|
|
|
|
test("discord-dm carries nonce and enforce_nonce when asked", async () => {
|
|
const dir = await mkdtemp(resolve(tmpdir(), "wn-bridge-"));
|
|
const stub = await fetchStub(dir);
|
|
const env = { ...process.env, NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" };
|
|
const dm = (...extra) => spawnSync(process.execPath, [BRIDGE, "discord-dm", "--recipient-id", "1", "--content", "hi", "--cultmesh-command-id", "cmd", "--receipt-store", resolve(dir, "r.cc"), ...extra], { encoding: "utf8", env });
|
|
|
|
const with_nonce = dm("--nonce", "abc123");
|
|
assert.equal(with_nonce.status, 0, with_nonce.stderr);
|
|
let posted = (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/channels/dm-channel/messages"));
|
|
assert.equal(posted.at(-1).body.nonce, "abc123");
|
|
assert.equal(posted.at(-1).body.enforce_nonce, true);
|
|
assert.deepEqual(posted.at(-1).body.allowed_mentions, { parse: [] }, "no mention is ever parsed");
|
|
|
|
const without = dm();
|
|
assert.equal(without.status, 0, without.stderr);
|
|
posted = (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/channels/dm-channel/messages"));
|
|
assert.ok(!("nonce" in posted.at(-1).body) && !("enforce_nonce" in posted.at(-1).body));
|
|
|
|
const before = (await fetches(stub.log)).length;
|
|
const tooLong = dm("--nonce", "x".repeat(26));
|
|
assert.notEqual(tooLong.status, 0);
|
|
assert.equal((await fetches(stub.log)).length, before, "an over-long nonce posts nothing");
|
|
});
|
|
|
|
test("discord-dm exits 75 only before delivery", async () => {
|
|
const dir = await mkdtemp(resolve(tmpdir(), "wn-exit75-"));
|
|
const stub = await fetchStub(dir);
|
|
const dm = (behavior, { token = "test-token-not-real", recipient = "1" } = {}) => {
|
|
const env = { ...process.env, NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: token, DISCORD_BOT_TOKEN: token };
|
|
if (behavior) env.STUB_BEHAVIOR = behavior;
|
|
if (!token) { delete env.BIFROST_DISCORD_BOT_TOKEN; delete env.DISCORD_BOT_TOKEN; }
|
|
const args = ["discord-dm", ...(recipient ? ["--recipient-id", recipient] : []), "--content", "hi", "--cultmesh-command-id", "cmd", "--receipt-store", resolve(dir, "r.cc")];
|
|
return spawnSync(process.execPath, [BRIDGE, ...args], { encoding: "utf8", env });
|
|
};
|
|
const messagePosts = async () => (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages")).length;
|
|
|
|
assert.equal(dm(undefined, { token: "" }).status, 75, "missing token");
|
|
assert.equal(dm(undefined, { recipient: "" }).status, 75, "missing argument");
|
|
assert.equal(dm("dm-channel-500").status, 75, "DM channel failure");
|
|
assert.equal(await messagePosts(), 0, "nothing was posted by any of those");
|
|
assert.equal(dm("message-400").status, 75, "a 4xx answer to the message POST");
|
|
assert.equal(await messagePosts(), 1);
|
|
for (const status of [500, 502, 503, 429]) {
|
|
const before = await messagePosts();
|
|
assert.equal(dm(`message-${status}`).status, 75, `a ${status} answer is retryable, not unknown`);
|
|
assert.equal(await messagePosts(), before + 1);
|
|
}
|
|
const lost = dm("message-network-error");
|
|
assert.equal(lost.status, 1, "a network error after the message POST was sent is not exit 75");
|
|
assert.equal(await messagePosts(), 6);
|
|
assert.equal(dm().status, 0);
|
|
});
|
|
|
|
test("a 5xx or 429 on the message POST is retried with the same nonce", async () => {
|
|
for (const status of [502, 429]) {
|
|
const w = await world(`wn-${status}-`);
|
|
const stub = await fetchStub(w.dir);
|
|
const env = { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" };
|
|
const messages = async () => (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages"));
|
|
for (let tick = 1; tick <= 2; tick += 1) {
|
|
const result = w.run({ bridge: BRIDGE, env: { ...env, ...at(tick === 1 ? 0 : 1), STUB_BEHAVIOR: `message-${status}` } });
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.deepEqual((await w.status()).rows.map((row) => `${row.status}:${row.attempts}`), [`failed:${tick}`, `failed:${tick}`]);
|
|
}
|
|
const sent = await messages();
|
|
assert.equal(sent.length, 4, "two openings, each tried twice");
|
|
const nonces = Map.groupBy(sent, (entry) => entry.body.content);
|
|
assert.equal(nonces.size, 2);
|
|
for (const group of nonces.values()) assert.equal(new Set(group.map((entry) => entry.body.nonce)).size, 1, "a retry reuses its nonce");
|
|
assert.equal(w.run({ bridge: BRIDGE, env: { ...env, ...at(3) } }).status, 0);
|
|
assert.deepEqual((await w.status()).rows.map((row) => row.status), ["completed", "completed", "completed"]);
|
|
}
|
|
});
|
|
|
|
test("every subject Idunn accepts is delivered, inside backticks", async () => {
|
|
const good = ["gamecult/odin", "odin:store", "_odin", "a".repeat(256), "Odin.v2-x"];
|
|
const bad = ["a".repeat(257), "has space", "back`tick", "new\nline", "", "ün"];
|
|
const w = await world("wn-subject-", { incidents: null });
|
|
await variantStore(w.incidents, (tuples) => {
|
|
return [...good, ...bad].map((subject, index) => {
|
|
const opened = 1_700_000_200_000 + index;
|
|
return ["idunn.operator_incident.v1", `continuity-exhausted:${subject}:${opened}`, "continuity-exhausted", subject, opened, null, null];
|
|
});
|
|
});
|
|
const stub = await fetchStub(w.dir);
|
|
const result = w.run({ bridge: BRIDGE, env: { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" } });
|
|
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: bad.length, unknown: 0, failing: 0 });
|
|
const contents = (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages")).map((entry) => entry.body.content).sort();
|
|
const at = (index) => new Date(1_700_000_200_000 + index).toISOString();
|
|
assert.deepEqual(contents, good.map((subject, index) => OPENED_TEXT(subject, at(index))).sort());
|
|
for (const content of contents) assert.match(content, /on `[A-Za-z0-9_.:/-]{1,256}` at /);
|
|
});
|
|
|
|
test("an out-of-range time refuses the record without aborting the run", async () => {
|
|
const w = await world("wn-time-", { incidents: null });
|
|
await variantStore(w.incidents, (tuples) => {
|
|
const [closed, open] = tuples;
|
|
const hugeClose = [...closed];
|
|
hugeClose[5] = 8.7e15;
|
|
hugeClose[1] = closed[1];
|
|
const hugeOpen = ["idunn.operator_incident.v1", "continuity-exhausted:huge-open:8700000000000000", "continuity-exhausted", "huge-open", 8.7e15, null, null];
|
|
const edge = ["idunn.operator_incident.v1", "continuity-exhausted:edge:8640000000000000", "continuity-exhausted", "edge", 8.64e15, null, null];
|
|
return [hugeClose, hugeOpen, edge, open];
|
|
});
|
|
const result = w.run();
|
|
assert.equal(result.stderr, "", "no unexpected-error");
|
|
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: 2, unknown: 0, failing: 0 });
|
|
assert.notEqual(result.status, 0, "refused records fail the run");
|
|
assert.deepEqual((await w.posts()).map((post) => opt(post, "source-id")).sort(), ["continuity-exhausted:edge:8640000000000000", OPEN_KEY]);
|
|
assert.ok((await w.status()).rows.every((row) => row.status === "completed"));
|
|
});
|
|
|
|
test("the reader posts through the real bridge with its nonce", async () => {
|
|
const w = await world("wn-real-bridge-");
|
|
const stub = await fetchStub(w.dir);
|
|
const result = w.run({ bridge: BRIDGE, env: { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" } });
|
|
assert.equal(result.status, 0, result.stderr);
|
|
const posted = (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages"));
|
|
assert.deepEqual(posted.map((entry) => entry.body.content).sort(), [CLOSED_TEXT, OPENED_TEXT("closed-target", T_OPEN), OPENED_TEXT("open-target", T_CLOSE)].sort());
|
|
for (const entry of posted) {
|
|
assert.equal(entry.body.nonce.length, 25);
|
|
assert.equal(entry.body.enforce_nonce, true);
|
|
}
|
|
assert.ok(existsSync(w.receipts), "the bridge wrote its crossing receipts to the reader's receipt store");
|
|
assert.deepEqual((await w.status()).rows.map((row) => row.status), ["completed", "completed", "completed"]);
|
|
assert.equal((await w.status()).status, 0, "status exits zero when nothing is unknown or failing");
|
|
});
|
|
|