Watchdog notice reader: a failed send is retried forever with capped backoff

Ruling notice-retry-policy. The five-attempt cap and its terminal
"exhausted" state are gone. A failed entry is eligible again once
min(2^(attempts-1), 60) minutes have passed since its last journal
update; a run inside the delay skips it without spawning the bridge.
Every attempt keeps the same nonce. The exit status is non-zero while
any entry is unknown or has failed 3 or more times, and retries go on.
status shows nextEligibleAt for failed entries.

The clock is one now() that WATCHDOG_NOTICE_NOW_MS can override, like
the bridge timeout override, so tests drive the backoff without sleeping.
The process summary field "exhausted" is renamed "failing". A journal
entry whose updatedAt is not a date is now refused as
journal-entry-invalid, since the delay is measured from it.

All commits build.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-01 01:48:44 +02:00
commit 69e12e3867
3 changed files with 116 additions and 37 deletions

View file

@ -22,6 +22,15 @@ node tools/watchdog-notice.mjs status --journal-store J
- `BIFROST_DISCORD_BOT_TOKEN` in the environment, read only by
`bifrost-bridge.mjs`, which the reader spawns.
## Retry
A notice whose send failed (bridge exit 75) is never dropped. It is eligible
again once `min(2^(attempts-1), 60)` minutes have passed since its last attempt
in the journal: 1, 2, 4, 8, 16, 32, then every 60 minutes. A run that finds it
inside its delay skips it without spawning the bridge. Every attempt carries the
same nonce, so a Discord outage of any length costs a late notice, not a lost
one, and a retry that succeeds is the one post.
## Behaviour
- P is Idunn's `idunn.operator_incident.v1` store. The reader reads it without a
@ -31,7 +40,7 @@ node tools/watchdog-notice.mjs status --journal-store J
- J is Bifrost's journal, `bifrost.watchdog_notice_execution.v1`, keyed
`<incident_key>#opened` or `<incident_key>#closed`. A `running` entry found at
start becomes `unknown` and is never sent again. A `failed` entry is retried on
later runs with the same nonce, up to five attempts. The journal is never
later runs with the same nonce, never given up on (see Retry). The journal is never
pruned: what Bifrost has sent is Bifrost's memory, whatever P holds.
- The bridge (`bifrost-bridge.mjs discord-dm`) exits 75 for argument or token
errors, a failure opening the DM channel, or any non-2xx answer to the message
@ -43,9 +52,11 @@ node tools/watchdog-notice.mjs status --journal-store J
`failed` for exit 75 only. Any other non-success without a message id (another exit code,
a signal, the spawn timeout, unreadable output) is `unknown`: never sent again,
counted by the exit status.
- The exit status is non-zero while any entry is `unknown` or out of attempts, or
while P holds a record of this schema that breaks its contract. systemd shows
that; `status` lists the entries.
- The exit status is non-zero while any entry is `unknown` or has failed three or
more times, or while P holds a record of this schema that breaks its contract.
systemd shows that; `status` lists the entries, with `attempts` and, for a
`failed` entry, `nextEligibleAt`. The non-zero status is the signal only: the
entry keeps being retried.
- Notice text is built from `condition`, `subject` and ISO UTC times only. The
subject follows Idunn's `require_id` (1-256 bytes of `[A-Za-z0-9-_.:/]`) and
is placed in inline code. A time outside the ECMAScript Date range refuses the

View file

@ -77,6 +77,11 @@ async function world(prefix, { incidents = FIXTURE, mode = "ok" } = {}) {
return w;
}
// The reader's clock is WATCHDOG_NOTICE_NOW_MS; a test drives the backoff by
// naming the minute (from T0) each run happens at.
const T0 = 1_800_000_000_000;
const at = (minutes) => ({ WATCHDOG_NOTICE_NOW_MS: String(T0 + Math.round(minutes * 60_000)) });
function run(w, { env = {}, bridge = w.bridge } = {}) {
return spawnSync(process.execPath, [TOOL, "process", "--incident-store", w.incidents, "--journal-store", w.journal, "--receipt-store", w.receipts, "--bridge-cli", bridge], {
encoding: "utf8",
@ -144,13 +149,14 @@ test("closure is posted once and only after a completed opening", async () => {
// An opening that never completes blocks its closure, now and later.
const failing = await world("wn-closure-failed-", { mode: "fail" });
for (let run = 0; run < 5; run += 1) failing.run();
await failing.setMode("ok");
failing.run();
failing.run();
for (const minute of [0, 1, 3, 7, 15]) failing.run({ env: at(minute) });
assert.equal((await failing.posts()).filter((post) => notice(post) === "closed").length, 0);
const rows = (await failing.status()).rows;
assert.deepEqual(rows.map((row) => `${row.notice}:${row.status}:${row.attempts}`), ["opened:failed:5", "opened:failed:5"]);
await failing.setMode("ok");
assert.equal(failing.run({ env: at(31) }).status, 0);
assert.equal(failing.run({ env: at(32) }).status, 0);
assert.deepEqual((await failing.posts()).filter((post) => notice(post) === "closed").map((post) => opt(post, "source-id")), [CLOSED_KEY], "the closure follows the completed opening, once");
const crashed = await world("wn-closure-unknown-", { mode: "crash" });
assert.equal(crashed.run().status, null, "the reader died mid-send");
@ -176,26 +182,75 @@ test("a running entry becomes unknown and is never re-posted", async () => {
assert.equal((await w.posts()).length, posts.length);
});
test("only a bridge exit 75 is retried, with the same nonce, up to five attempts", async () => {
const backoffMinutes = [0, 1, 3, 7, 15, 31, 63, 123, 183];
test("a failed notice is retried after its backoff and never exhausted", async () => {
const w = await world("wn-retry-", { mode: "fail" });
const exits = [];
for (let run = 0; run < 5; run += 1) exits.push(w.run().status);
assert.deepEqual(exits, [0, 0, 0, 0, 1], "non-zero once attempts are exhausted");
assert.deepEqual((await w.status()).rows.map((row) => row.lastError), ["bridge-not-sent", "bridge-not-sent"]);
// Run at every expected spawn minute and one minute before it: only the former may spawn.
const probes = [...new Set(backoffMinutes.flatMap((minute) => [minute - 1, minute]).filter((minute) => minute >= 0))].sort((a, b) => a - b);
const spawned = [];
for (const minute of probes) {
const before = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY).length;
w.run({ env: at(minute) });
const after = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY).length;
if (after > before) spawned.push(minute);
}
assert.deepEqual(spawned, backoffMinutes, "attempts at 1, 2, 4, 8, 16, 32 then every 60 minutes after the last attempt");
const open = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY);
assert.equal(open.length, 5);
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1);
const before = (await w.posts()).length;
const again = w.run();
assert.notEqual(again.status, 0);
assert.equal((await w.posts()).length, before, "no spawn after the fifth attempt");
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1, "every attempt carries the same nonce");
const rows = (await w.status()).rows;
assert.deepEqual(rows.map((row) => `${row.status}:${row.attempts}:${row.lastError}`), ["failed:9:bridge-not-sent", "failed:9:bridge-not-sent"], "never a terminal state");
const final = w.run({ env: at(183) });
const status = await w.status();
assert.notEqual(final.status, 0);
assert.notEqual(status.status, 0);
for (const text of [again.stdout, again.stderr, status.stdout, await readFile(w.journal, "latin1")]) {
for (const text of [final.stdout, final.stderr, status.stdout, await readFile(w.journal, "latin1")]) {
assert.ok(!text.includes("RECIPIENTCANARY"), "bridge output and recipient never reach the reader's output or journal");
}
});
test("a failed notice is not retried before its delay", async () => {
const w = await world("wn-delay-", { mode: "fail" });
const count = async () => (await w.posts()).length;
w.run({ env: at(0) });
assert.equal(await count(), 2, "both openings tried once");
w.run({ env: at(0.99) });
w.run({ env: at(0) });
assert.equal(await count(), 2, "a run inside the delay spawns nothing");
const row = (await w.status()).rows[0];
assert.equal(row.attempts, 1);
assert.equal(row.nextEligibleAt, new Date(T0 + 60_000).toISOString(), "status names when the entry is next eligible");
w.run({ env: at(1) });
assert.equal(await count(), 4, "eligible once the delay has passed");
assert.equal((await w.status()).rows[0].nextEligibleAt, new Date(T0 + 3 * 60_000).toISOString());
});
test("the unit signals from the third failed attempt while still retrying", async () => {
const w = await world("wn-signal-", { mode: "fail" });
const exits = [];
for (const minute of [0, 1, 3]) exits.push(w.run({ env: at(minute) }).status);
assert.deepEqual(exits, [0, 0, 1], "zero after attempts 1 and 2, non-zero from 3");
assert.equal((await w.status()).status, 1);
const before = (await w.posts()).length;
assert.equal(w.run({ env: at(7) }).status, 1);
assert.equal((await w.posts()).length, before + 2, "attempt 4 still spawns");
assert.ok((await w.status()).rows.every((row) => row.status === "failed" && row.attempts === 4));
});
test("a notice that finally sends after many failures posts once", async () => {
const w = await world("wn-finally-", { mode: "fail" });
for (const minute of [0, 1, 3, 7, 15]) w.run({ env: at(minute) });
await w.setMode("ok");
assert.equal(w.run({ env: at(31) }).status, 0);
const open = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY);
assert.equal(open.length, 6, "five failures and one success");
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1, "the success carries the same nonce");
const before = (await w.posts()).length;
for (const minute of [32, 100, 1000]) assert.equal(w.run({ env: at(minute) }).status, 0);
assert.equal((await w.posts()).length, before, "completed is never posted again");
assert.deepEqual((await w.status()).rows.map((row) => `${row.status}:${row.attempts}`), ["completed:6", "completed:6", "completed:6"]);
});
test("the reader never writes or locks the incident store", async () => {
const w = await world("wn-readonly-");
const before = { bytes: await readFile(w.incidents), mtime: (await stat(w.incidents)).mtimeMs };
@ -217,7 +272,7 @@ test("notice content carries no sensitive text", async () => {
});
const result = w.run();
assert.notEqual(result.status, 0, "refused records fail the run");
assert.deepEqual(JSON.parse(result.stdout), { skipped: 1, refused: 2, unknown: 0, exhausted: 0 });
assert.deepEqual(JSON.parse(result.stdout), { skipped: 1, refused: 2, unknown: 0, failing: 0 });
const posts = await w.posts();
assert.deepEqual(posts.map((post) => opt(post, "source-id")), [CLOSED_KEY, CLOSED_KEY]);
for (const post of posts) assert.match(post.content, /^Idunn incident (opened: continuity-exhausted on `closed-target` at \d{4}-[\d-]+T[\d:.]+Z|closed \(recovered\): continuity-exhausted on `closed-target`, opened [\dTZ:.-]+, closed [\dTZ:.-]+)$/);
@ -390,7 +445,7 @@ test("a 5xx or 429 on the message POST is retried with the same nonce", async ()
const env = { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" };
const messages = async () => (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages"));
for (let tick = 1; tick <= 2; tick += 1) {
const result = w.run({ bridge: BRIDGE, env: { ...env, STUB_BEHAVIOR: `message-${status}` } });
const result = w.run({ bridge: BRIDGE, env: { ...env, ...at(tick === 1 ? 0 : 1), STUB_BEHAVIOR: `message-${status}` } });
assert.equal(result.status, 0, result.stderr);
assert.deepEqual((await w.status()).rows.map((row) => `${row.status}:${row.attempts}`), [`failed:${tick}`, `failed:${tick}`]);
}
@ -399,7 +454,7 @@ test("a 5xx or 429 on the message POST is retried with the same nonce", async ()
const nonces = Map.groupBy(sent, (entry) => entry.body.content);
assert.equal(nonces.size, 2);
for (const group of nonces.values()) assert.equal(new Set(group.map((entry) => entry.body.nonce)).size, 1, "a retry reuses its nonce");
assert.equal(w.run({ bridge: BRIDGE, env }).status, 0);
assert.equal(w.run({ bridge: BRIDGE, env: { ...env, ...at(3) } }).status, 0);
assert.deepEqual((await w.status()).rows.map((row) => row.status), ["completed", "completed", "completed"]);
}
});
@ -416,7 +471,7 @@ test("every subject Idunn accepts is delivered, inside backticks", async () => {
});
const stub = await fetchStub(w.dir);
const result = w.run({ bridge: BRIDGE, env: { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" } });
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: bad.length, unknown: 0, exhausted: 0 });
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: bad.length, unknown: 0, failing: 0 });
const contents = (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages")).map((entry) => entry.body.content).sort();
const at = (index) => new Date(1_700_000_200_000 + index).toISOString();
assert.deepEqual(contents, good.map((subject, index) => OPENED_TEXT(subject, at(index))).sort());
@ -436,7 +491,7 @@ test("an out-of-range time refuses the record without aborting the run", async (
});
const result = w.run();
assert.equal(result.stderr, "", "no unexpected-error");
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: 2, unknown: 0, exhausted: 0 });
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: 2, unknown: 0, failing: 0 });
assert.notEqual(result.status, 0, "refused records fail the run");
assert.deepEqual((await w.posts()).map((post) => opt(post, "source-id")).sort(), ["continuity-exhausted:edge:8640000000000000", OPEN_KEY]);
assert.ok((await w.status()).rows.every((row) => row.status === "completed"));
@ -455,6 +510,6 @@ test("the reader posts through the real bridge with its nonce", async () => {
}
assert.ok(existsSync(w.receipts), "the bridge wrote its crossing receipts to the reader's receipt store");
assert.deepEqual((await w.status()).rows.map((row) => row.status), ["completed", "completed", "completed"]);
assert.equal((await w.status()).status, 0, "status exits zero when nothing is unknown or exhausted");
assert.equal((await w.status()).status, 0, "status exits zero when nothing is unknown or failing");
});

View file

@ -27,9 +27,15 @@ const cult = resolve(process.env.VOIDBOT_CULTLIB_ROOT || resolve(root, "..", "Cu
const INCIDENT_SCHEMA = "idunn.operator_incident.v1";
const JOURNAL_TYPE = "bifrost.watchdog_notice_execution";
const JOURNAL_SCHEMA = "bifrost.watchdog_notice_execution.v1";
const MAX_ATTEMPTS = 5;
// The override exists so a test can reach the timeout branch without a 60 s wait.
// A failed notice is retried forever: 1, 2, 4, ... minutes after its last
// attempt, capped at 60 (ruling notice-retry-policy). From FAILING_ATTEMPTS
// failures on, the exit status is the delivery-failure signal; retries go on.
const MAX_DELAY_MINUTES = 60;
const FAILING_ATTEMPTS = 3;
// The overrides exist so a test can reach the timeout branch without a 60 s wait
// and can drive the backoff without sleeping.
const BRIDGE_TIMEOUT_MS = Number(process.env.WATCHDOG_NOTICE_BRIDGE_TIMEOUT_MS) || 60_000;
const clockMs = () => Number(process.env.WATCHDOG_NOTICE_NOW_MS) || Date.now();
const STATUSES = ["running", "completed", "failed", "unknown"];
const NAME = /^[a-z][a-z-]{0,63}$/;
// Idunn's require_id (GameCult/Idunn src/control_plane.rs:9122-9131 at 7f7528b):
@ -82,7 +88,7 @@ async function processIncidents(options) {
const outcome = await sendNotice({ bridgeCli, recipientId, receiptStore, key, nonce, record, content });
await journal.put({ ...base, messageId: outcome.messageId ?? "", status: outcome.status, lastError: outcome.error ?? "", updatedAt: now() });
};
const retryable = (entry) => !entry || (entry.status === "failed" && entry.attempts < MAX_ATTEMPTS);
const retryable = (entry) => !entry || (entry.status === "failed" && !(clockMs() < nextEligibleMs(entry)));
const entryFor = (record, notice) => journal.cache.get(journal.definition, `${record.incidentKey}#${notice}`);
for (const record of records) {
@ -94,9 +100,9 @@ async function processIncidents(options) {
const entries = journal.cache.getAll(journal.definition);
const unknown = entries.filter((entry) => entry.status === "unknown").length;
const exhausted = entries.filter(isExhausted).length;
process.stdout.write(`${JSON.stringify({ skipped, refused, unknown, exhausted })}\n`);
return unknown + exhausted + refused > 0 ? 1 : 0;
const failing = entries.filter(isFailing).length;
process.stdout.write(`${JSON.stringify({ skipped, refused, unknown, failing })}\n`);
return unknown + failing + refused > 0 ? 1 : 0;
}
async function status(options) {
@ -104,14 +110,20 @@ async function status(options) {
const journal = await openJournal(runtime, requireOption(options, "journal-store"));
const entries = journal.cache.getAll(journal.definition);
const rows = entries
.map(({ incidentKey, notice, status, attempts, updatedAt, lastError }) => ({ incidentKey, notice, status, attempts, updatedAt, lastError }))
.map((entry) => ({
incidentKey: entry.incidentKey, notice: entry.notice, status: entry.status, attempts: entry.attempts,
updatedAt: entry.updatedAt, lastError: entry.lastError,
nextEligibleAt: entry.status === "failed" ? new Date(nextEligibleMs(entry)).toISOString() : null,
}))
.sort((a, b) => (a.incidentKey + a.notice).localeCompare(b.incidentKey + b.notice));
process.stdout.write(`${JSON.stringify(rows, null, 2)}\n`);
return entries.some((entry) => entry.status === "unknown" || isExhausted(entry)) ? 1 : 0;
return entries.some((entry) => entry.status === "unknown" || isFailing(entry)) ? 1 : 0;
}
const isExhausted = (entry) => entry.status === "failed" && entry.attempts >= MAX_ATTEMPTS;
const now = () => new Date().toISOString();
const isFailing = (entry) => entry.status === "failed" && entry.attempts >= FAILING_ATTEMPTS;
// Measured from the failed entry's last attempt in the journal.
const nextEligibleMs = (entry) => Date.parse(entry.updatedAt) + Math.min(2 ** (entry.attempts - 1), MAX_DELAY_MINUTES) * 60_000;
const now = () => new Date(clockMs()).toISOString();
// Reads P unlocked. An absent file is "nothing to do" (records: null). A
// record of another schema is counted as skipped; a record of this schema
@ -240,7 +252,8 @@ function parseEntry(value) {
&& (value.notice === "opened" || value.notice === "closed")
&& STATUSES.includes(value.status)
&& Number.isSafeInteger(value.attempts) && value.attempts >= 0
&& ["nonce", "messageId", "lastError", "updatedAt"].every((field) => typeof value[field] === "string");
&& ["nonce", "messageId", "lastError", "updatedAt"].every((field) => typeof value[field] === "string")
&& Number.isFinite(Date.parse(value.updatedAt));
if (!ok) throw fail("journal-entry-invalid");
return value;
}