Watchdog notice reader: a failed send is retried forever with capped backoff
Ruling notice-retry-policy. The five-attempt cap and its terminal "exhausted" state are gone. A failed entry is eligible again once min(2^(attempts-1), 60) minutes have passed since its last journal update; a run inside the delay skips it without spawning the bridge. Every attempt keeps the same nonce. The exit status is non-zero while any entry is unknown or has failed 3 or more times, and retries go on. status shows nextEligibleAt for failed entries. The clock is one now() that WATCHDOG_NOTICE_NOW_MS can override, like the bridge timeout override, so tests drive the backoff without sleeping. The process summary field "exhausted" is renamed "failing". A journal entry whose updatedAt is not a date is now refused as journal-entry-invalid, since the delay is measured from it. All commits build. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
68fb44aeb1
commit
69e12e3867
3 changed files with 116 additions and 37 deletions
|
|
@ -22,6 +22,15 @@ node tools/watchdog-notice.mjs status --journal-store J
|
|||
- `BIFROST_DISCORD_BOT_TOKEN` in the environment, read only by
|
||||
`bifrost-bridge.mjs`, which the reader spawns.
|
||||
|
||||
## Retry
|
||||
|
||||
A notice whose send failed (bridge exit 75) is never dropped. It is eligible
|
||||
again once `min(2^(attempts-1), 60)` minutes have passed since its last attempt
|
||||
in the journal: 1, 2, 4, 8, 16, 32, then every 60 minutes. A run that finds it
|
||||
inside its delay skips it without spawning the bridge. Every attempt carries the
|
||||
same nonce, so a Discord outage of any length costs a late notice, not a lost
|
||||
one, and a retry that succeeds is the one post.
|
||||
|
||||
## Behaviour
|
||||
|
||||
- P is Idunn's `idunn.operator_incident.v1` store. The reader reads it without a
|
||||
|
|
@ -31,7 +40,7 @@ node tools/watchdog-notice.mjs status --journal-store J
|
|||
- J is Bifrost's journal, `bifrost.watchdog_notice_execution.v1`, keyed
|
||||
`<incident_key>#opened` or `<incident_key>#closed`. A `running` entry found at
|
||||
start becomes `unknown` and is never sent again. A `failed` entry is retried on
|
||||
later runs with the same nonce, up to five attempts. The journal is never
|
||||
later runs with the same nonce, never given up on (see Retry). The journal is never
|
||||
pruned: what Bifrost has sent is Bifrost's memory, whatever P holds.
|
||||
- The bridge (`bifrost-bridge.mjs discord-dm`) exits 75 for argument or token
|
||||
errors, a failure opening the DM channel, or any non-2xx answer to the message
|
||||
|
|
@ -43,9 +52,11 @@ node tools/watchdog-notice.mjs status --journal-store J
|
|||
`failed` for exit 75 only. Any other non-success without a message id (another exit code,
|
||||
a signal, the spawn timeout, unreadable output) is `unknown`: never sent again,
|
||||
counted by the exit status.
|
||||
- The exit status is non-zero while any entry is `unknown` or out of attempts, or
|
||||
while P holds a record of this schema that breaks its contract. systemd shows
|
||||
that; `status` lists the entries.
|
||||
- The exit status is non-zero while any entry is `unknown` or has failed three or
|
||||
more times, or while P holds a record of this schema that breaks its contract.
|
||||
systemd shows that; `status` lists the entries, with `attempts` and, for a
|
||||
`failed` entry, `nextEligibleAt`. The non-zero status is the signal only: the
|
||||
entry keeps being retried.
|
||||
- Notice text is built from `condition`, `subject` and ISO UTC times only. The
|
||||
subject follows Idunn's `require_id` (1-256 bytes of `[A-Za-z0-9-_.:/]`) and
|
||||
is placed in inline code. A time outside the ECMAScript Date range refuses the
|
||||
|
|
|
|||
|
|
@ -77,6 +77,11 @@ async function world(prefix, { incidents = FIXTURE, mode = "ok" } = {}) {
|
|||
return w;
|
||||
}
|
||||
|
||||
// The reader's clock is WATCHDOG_NOTICE_NOW_MS; a test drives the backoff by
|
||||
// naming the minute (from T0) each run happens at.
|
||||
const T0 = 1_800_000_000_000;
|
||||
const at = (minutes) => ({ WATCHDOG_NOTICE_NOW_MS: String(T0 + Math.round(minutes * 60_000)) });
|
||||
|
||||
function run(w, { env = {}, bridge = w.bridge } = {}) {
|
||||
return spawnSync(process.execPath, [TOOL, "process", "--incident-store", w.incidents, "--journal-store", w.journal, "--receipt-store", w.receipts, "--bridge-cli", bridge], {
|
||||
encoding: "utf8",
|
||||
|
|
@ -144,13 +149,14 @@ test("closure is posted once and only after a completed opening", async () => {
|
|||
|
||||
// An opening that never completes blocks its closure, now and later.
|
||||
const failing = await world("wn-closure-failed-", { mode: "fail" });
|
||||
for (let run = 0; run < 5; run += 1) failing.run();
|
||||
await failing.setMode("ok");
|
||||
failing.run();
|
||||
failing.run();
|
||||
for (const minute of [0, 1, 3, 7, 15]) failing.run({ env: at(minute) });
|
||||
assert.equal((await failing.posts()).filter((post) => notice(post) === "closed").length, 0);
|
||||
const rows = (await failing.status()).rows;
|
||||
assert.deepEqual(rows.map((row) => `${row.notice}:${row.status}:${row.attempts}`), ["opened:failed:5", "opened:failed:5"]);
|
||||
await failing.setMode("ok");
|
||||
assert.equal(failing.run({ env: at(31) }).status, 0);
|
||||
assert.equal(failing.run({ env: at(32) }).status, 0);
|
||||
assert.deepEqual((await failing.posts()).filter((post) => notice(post) === "closed").map((post) => opt(post, "source-id")), [CLOSED_KEY], "the closure follows the completed opening, once");
|
||||
|
||||
const crashed = await world("wn-closure-unknown-", { mode: "crash" });
|
||||
assert.equal(crashed.run().status, null, "the reader died mid-send");
|
||||
|
|
@ -176,26 +182,75 @@ test("a running entry becomes unknown and is never re-posted", async () => {
|
|||
assert.equal((await w.posts()).length, posts.length);
|
||||
});
|
||||
|
||||
test("only a bridge exit 75 is retried, with the same nonce, up to five attempts", async () => {
|
||||
const backoffMinutes = [0, 1, 3, 7, 15, 31, 63, 123, 183];
|
||||
|
||||
test("a failed notice is retried after its backoff and never exhausted", async () => {
|
||||
const w = await world("wn-retry-", { mode: "fail" });
|
||||
const exits = [];
|
||||
for (let run = 0; run < 5; run += 1) exits.push(w.run().status);
|
||||
assert.deepEqual(exits, [0, 0, 0, 0, 1], "non-zero once attempts are exhausted");
|
||||
assert.deepEqual((await w.status()).rows.map((row) => row.lastError), ["bridge-not-sent", "bridge-not-sent"]);
|
||||
// Run at every expected spawn minute and one minute before it: only the former may spawn.
|
||||
const probes = [...new Set(backoffMinutes.flatMap((minute) => [minute - 1, minute]).filter((minute) => minute >= 0))].sort((a, b) => a - b);
|
||||
const spawned = [];
|
||||
for (const minute of probes) {
|
||||
const before = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY).length;
|
||||
w.run({ env: at(minute) });
|
||||
const after = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY).length;
|
||||
if (after > before) spawned.push(minute);
|
||||
}
|
||||
assert.deepEqual(spawned, backoffMinutes, "attempts at 1, 2, 4, 8, 16, 32 then every 60 minutes after the last attempt");
|
||||
const open = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY);
|
||||
assert.equal(open.length, 5);
|
||||
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1);
|
||||
const before = (await w.posts()).length;
|
||||
const again = w.run();
|
||||
assert.notEqual(again.status, 0);
|
||||
assert.equal((await w.posts()).length, before, "no spawn after the fifth attempt");
|
||||
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1, "every attempt carries the same nonce");
|
||||
const rows = (await w.status()).rows;
|
||||
assert.deepEqual(rows.map((row) => `${row.status}:${row.attempts}:${row.lastError}`), ["failed:9:bridge-not-sent", "failed:9:bridge-not-sent"], "never a terminal state");
|
||||
const final = w.run({ env: at(183) });
|
||||
const status = await w.status();
|
||||
assert.notEqual(final.status, 0);
|
||||
assert.notEqual(status.status, 0);
|
||||
for (const text of [again.stdout, again.stderr, status.stdout, await readFile(w.journal, "latin1")]) {
|
||||
for (const text of [final.stdout, final.stderr, status.stdout, await readFile(w.journal, "latin1")]) {
|
||||
assert.ok(!text.includes("RECIPIENTCANARY"), "bridge output and recipient never reach the reader's output or journal");
|
||||
}
|
||||
});
|
||||
|
||||
test("a failed notice is not retried before its delay", async () => {
|
||||
const w = await world("wn-delay-", { mode: "fail" });
|
||||
const count = async () => (await w.posts()).length;
|
||||
w.run({ env: at(0) });
|
||||
assert.equal(await count(), 2, "both openings tried once");
|
||||
w.run({ env: at(0.99) });
|
||||
w.run({ env: at(0) });
|
||||
assert.equal(await count(), 2, "a run inside the delay spawns nothing");
|
||||
const row = (await w.status()).rows[0];
|
||||
assert.equal(row.attempts, 1);
|
||||
assert.equal(row.nextEligibleAt, new Date(T0 + 60_000).toISOString(), "status names when the entry is next eligible");
|
||||
w.run({ env: at(1) });
|
||||
assert.equal(await count(), 4, "eligible once the delay has passed");
|
||||
assert.equal((await w.status()).rows[0].nextEligibleAt, new Date(T0 + 3 * 60_000).toISOString());
|
||||
});
|
||||
|
||||
test("the unit signals from the third failed attempt while still retrying", async () => {
|
||||
const w = await world("wn-signal-", { mode: "fail" });
|
||||
const exits = [];
|
||||
for (const minute of [0, 1, 3]) exits.push(w.run({ env: at(minute) }).status);
|
||||
assert.deepEqual(exits, [0, 0, 1], "zero after attempts 1 and 2, non-zero from 3");
|
||||
assert.equal((await w.status()).status, 1);
|
||||
const before = (await w.posts()).length;
|
||||
assert.equal(w.run({ env: at(7) }).status, 1);
|
||||
assert.equal((await w.posts()).length, before + 2, "attempt 4 still spawns");
|
||||
assert.ok((await w.status()).rows.every((row) => row.status === "failed" && row.attempts === 4));
|
||||
});
|
||||
|
||||
test("a notice that finally sends after many failures posts once", async () => {
|
||||
const w = await world("wn-finally-", { mode: "fail" });
|
||||
for (const minute of [0, 1, 3, 7, 15]) w.run({ env: at(minute) });
|
||||
await w.setMode("ok");
|
||||
assert.equal(w.run({ env: at(31) }).status, 0);
|
||||
const open = (await w.posts()).filter((post) => opt(post, "source-id") === OPEN_KEY);
|
||||
assert.equal(open.length, 6, "five failures and one success");
|
||||
assert.equal(new Set(open.map((post) => opt(post, "nonce"))).size, 1, "the success carries the same nonce");
|
||||
const before = (await w.posts()).length;
|
||||
for (const minute of [32, 100, 1000]) assert.equal(w.run({ env: at(minute) }).status, 0);
|
||||
assert.equal((await w.posts()).length, before, "completed is never posted again");
|
||||
assert.deepEqual((await w.status()).rows.map((row) => `${row.status}:${row.attempts}`), ["completed:6", "completed:6", "completed:6"]);
|
||||
});
|
||||
|
||||
test("the reader never writes or locks the incident store", async () => {
|
||||
const w = await world("wn-readonly-");
|
||||
const before = { bytes: await readFile(w.incidents), mtime: (await stat(w.incidents)).mtimeMs };
|
||||
|
|
@ -217,7 +272,7 @@ test("notice content carries no sensitive text", async () => {
|
|||
});
|
||||
const result = w.run();
|
||||
assert.notEqual(result.status, 0, "refused records fail the run");
|
||||
assert.deepEqual(JSON.parse(result.stdout), { skipped: 1, refused: 2, unknown: 0, exhausted: 0 });
|
||||
assert.deepEqual(JSON.parse(result.stdout), { skipped: 1, refused: 2, unknown: 0, failing: 0 });
|
||||
const posts = await w.posts();
|
||||
assert.deepEqual(posts.map((post) => opt(post, "source-id")), [CLOSED_KEY, CLOSED_KEY]);
|
||||
for (const post of posts) assert.match(post.content, /^Idunn incident (opened: continuity-exhausted on `closed-target` at \d{4}-[\d-]+T[\d:.]+Z|closed \(recovered\): continuity-exhausted on `closed-target`, opened [\dTZ:.-]+, closed [\dTZ:.-]+)$/);
|
||||
|
|
@ -390,7 +445,7 @@ test("a 5xx or 429 on the message POST is retried with the same nonce", async ()
|
|||
const env = { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" };
|
||||
const messages = async () => (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages"));
|
||||
for (let tick = 1; tick <= 2; tick += 1) {
|
||||
const result = w.run({ bridge: BRIDGE, env: { ...env, STUB_BEHAVIOR: `message-${status}` } });
|
||||
const result = w.run({ bridge: BRIDGE, env: { ...env, ...at(tick === 1 ? 0 : 1), STUB_BEHAVIOR: `message-${status}` } });
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.deepEqual((await w.status()).rows.map((row) => `${row.status}:${row.attempts}`), [`failed:${tick}`, `failed:${tick}`]);
|
||||
}
|
||||
|
|
@ -399,7 +454,7 @@ test("a 5xx or 429 on the message POST is retried with the same nonce", async ()
|
|||
const nonces = Map.groupBy(sent, (entry) => entry.body.content);
|
||||
assert.equal(nonces.size, 2);
|
||||
for (const group of nonces.values()) assert.equal(new Set(group.map((entry) => entry.body.nonce)).size, 1, "a retry reuses its nonce");
|
||||
assert.equal(w.run({ bridge: BRIDGE, env }).status, 0);
|
||||
assert.equal(w.run({ bridge: BRIDGE, env: { ...env, ...at(3) } }).status, 0);
|
||||
assert.deepEqual((await w.status()).rows.map((row) => row.status), ["completed", "completed", "completed"]);
|
||||
}
|
||||
});
|
||||
|
|
@ -416,7 +471,7 @@ test("every subject Idunn accepts is delivered, inside backticks", async () => {
|
|||
});
|
||||
const stub = await fetchStub(w.dir);
|
||||
const result = w.run({ bridge: BRIDGE, env: { NODE_OPTIONS: stub.options, BIFROST_DISCORD_BOT_TOKEN: "test-token-not-real" } });
|
||||
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: bad.length, unknown: 0, exhausted: 0 });
|
||||
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: bad.length, unknown: 0, failing: 0 });
|
||||
const contents = (await fetches(stub.log)).filter((entry) => entry.url.endsWith("/messages")).map((entry) => entry.body.content).sort();
|
||||
const at = (index) => new Date(1_700_000_200_000 + index).toISOString();
|
||||
assert.deepEqual(contents, good.map((subject, index) => OPENED_TEXT(subject, at(index))).sort());
|
||||
|
|
@ -436,7 +491,7 @@ test("an out-of-range time refuses the record without aborting the run", async (
|
|||
});
|
||||
const result = w.run();
|
||||
assert.equal(result.stderr, "", "no unexpected-error");
|
||||
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: 2, unknown: 0, exhausted: 0 });
|
||||
assert.deepEqual(JSON.parse(result.stdout), { skipped: 0, refused: 2, unknown: 0, failing: 0 });
|
||||
assert.notEqual(result.status, 0, "refused records fail the run");
|
||||
assert.deepEqual((await w.posts()).map((post) => opt(post, "source-id")).sort(), ["continuity-exhausted:edge:8640000000000000", OPEN_KEY]);
|
||||
assert.ok((await w.status()).rows.every((row) => row.status === "completed"));
|
||||
|
|
@ -455,6 +510,6 @@ test("the reader posts through the real bridge with its nonce", async () => {
|
|||
}
|
||||
assert.ok(existsSync(w.receipts), "the bridge wrote its crossing receipts to the reader's receipt store");
|
||||
assert.deepEqual((await w.status()).rows.map((row) => row.status), ["completed", "completed", "completed"]);
|
||||
assert.equal((await w.status()).status, 0, "status exits zero when nothing is unknown or exhausted");
|
||||
assert.equal((await w.status()).status, 0, "status exits zero when nothing is unknown or failing");
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -27,9 +27,15 @@ const cult = resolve(process.env.VOIDBOT_CULTLIB_ROOT || resolve(root, "..", "Cu
|
|||
const INCIDENT_SCHEMA = "idunn.operator_incident.v1";
|
||||
const JOURNAL_TYPE = "bifrost.watchdog_notice_execution";
|
||||
const JOURNAL_SCHEMA = "bifrost.watchdog_notice_execution.v1";
|
||||
const MAX_ATTEMPTS = 5;
|
||||
// The override exists so a test can reach the timeout branch without a 60 s wait.
|
||||
// A failed notice is retried forever: 1, 2, 4, ... minutes after its last
|
||||
// attempt, capped at 60 (ruling notice-retry-policy). From FAILING_ATTEMPTS
|
||||
// failures on, the exit status is the delivery-failure signal; retries go on.
|
||||
const MAX_DELAY_MINUTES = 60;
|
||||
const FAILING_ATTEMPTS = 3;
|
||||
// The overrides exist so a test can reach the timeout branch without a 60 s wait
|
||||
// and can drive the backoff without sleeping.
|
||||
const BRIDGE_TIMEOUT_MS = Number(process.env.WATCHDOG_NOTICE_BRIDGE_TIMEOUT_MS) || 60_000;
|
||||
const clockMs = () => Number(process.env.WATCHDOG_NOTICE_NOW_MS) || Date.now();
|
||||
const STATUSES = ["running", "completed", "failed", "unknown"];
|
||||
const NAME = /^[a-z][a-z-]{0,63}$/;
|
||||
// Idunn's require_id (GameCult/Idunn src/control_plane.rs:9122-9131 at 7f7528b):
|
||||
|
|
@ -82,7 +88,7 @@ async function processIncidents(options) {
|
|||
const outcome = await sendNotice({ bridgeCli, recipientId, receiptStore, key, nonce, record, content });
|
||||
await journal.put({ ...base, messageId: outcome.messageId ?? "", status: outcome.status, lastError: outcome.error ?? "", updatedAt: now() });
|
||||
};
|
||||
const retryable = (entry) => !entry || (entry.status === "failed" && entry.attempts < MAX_ATTEMPTS);
|
||||
const retryable = (entry) => !entry || (entry.status === "failed" && !(clockMs() < nextEligibleMs(entry)));
|
||||
const entryFor = (record, notice) => journal.cache.get(journal.definition, `${record.incidentKey}#${notice}`);
|
||||
|
||||
for (const record of records) {
|
||||
|
|
@ -94,9 +100,9 @@ async function processIncidents(options) {
|
|||
|
||||
const entries = journal.cache.getAll(journal.definition);
|
||||
const unknown = entries.filter((entry) => entry.status === "unknown").length;
|
||||
const exhausted = entries.filter(isExhausted).length;
|
||||
process.stdout.write(`${JSON.stringify({ skipped, refused, unknown, exhausted })}\n`);
|
||||
return unknown + exhausted + refused > 0 ? 1 : 0;
|
||||
const failing = entries.filter(isFailing).length;
|
||||
process.stdout.write(`${JSON.stringify({ skipped, refused, unknown, failing })}\n`);
|
||||
return unknown + failing + refused > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
async function status(options) {
|
||||
|
|
@ -104,14 +110,20 @@ async function status(options) {
|
|||
const journal = await openJournal(runtime, requireOption(options, "journal-store"));
|
||||
const entries = journal.cache.getAll(journal.definition);
|
||||
const rows = entries
|
||||
.map(({ incidentKey, notice, status, attempts, updatedAt, lastError }) => ({ incidentKey, notice, status, attempts, updatedAt, lastError }))
|
||||
.map((entry) => ({
|
||||
incidentKey: entry.incidentKey, notice: entry.notice, status: entry.status, attempts: entry.attempts,
|
||||
updatedAt: entry.updatedAt, lastError: entry.lastError,
|
||||
nextEligibleAt: entry.status === "failed" ? new Date(nextEligibleMs(entry)).toISOString() : null,
|
||||
}))
|
||||
.sort((a, b) => (a.incidentKey + a.notice).localeCompare(b.incidentKey + b.notice));
|
||||
process.stdout.write(`${JSON.stringify(rows, null, 2)}\n`);
|
||||
return entries.some((entry) => entry.status === "unknown" || isExhausted(entry)) ? 1 : 0;
|
||||
return entries.some((entry) => entry.status === "unknown" || isFailing(entry)) ? 1 : 0;
|
||||
}
|
||||
|
||||
const isExhausted = (entry) => entry.status === "failed" && entry.attempts >= MAX_ATTEMPTS;
|
||||
const now = () => new Date().toISOString();
|
||||
const isFailing = (entry) => entry.status === "failed" && entry.attempts >= FAILING_ATTEMPTS;
|
||||
// Measured from the failed entry's last attempt in the journal.
|
||||
const nextEligibleMs = (entry) => Date.parse(entry.updatedAt) + Math.min(2 ** (entry.attempts - 1), MAX_DELAY_MINUTES) * 60_000;
|
||||
const now = () => new Date(clockMs()).toISOString();
|
||||
|
||||
// Reads P unlocked. An absent file is "nothing to do" (records: null). A
|
||||
// record of another schema is counted as skipped; a record of this schema
|
||||
|
|
@ -240,7 +252,8 @@ function parseEntry(value) {
|
|||
&& (value.notice === "opened" || value.notice === "closed")
|
||||
&& STATUSES.includes(value.status)
|
||||
&& Number.isSafeInteger(value.attempts) && value.attempts >= 0
|
||||
&& ["nonce", "messageId", "lastError", "updatedAt"].every((field) => typeof value[field] === "string");
|
||||
&& ["nonce", "messageId", "lastError", "updatedAt"].every((field) => typeof value[field] === "string")
|
||||
&& Number.isFinite(Date.parse(value.updatedAt));
|
||||
if (!ok) throw fail("journal-entry-invalid");
|
||||
return value;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue