Record a failed refresh on the linked identity row so another process does not re-present the token

A failed or timed-out refresh attempt now writes refresh_failed_at under the row lock; a process
that takes the lock inside a 60 s backoff answers managed_credential_refresh_failed without calling
the provider. Rotation, revocation and re-link clear the marker. Closes the cross-process residual
that refresh-once-per-token left to a follow-up, and pins it on the two-pool rig.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Heimdall 2026-11-08 21:35:13 +00:00
commit f5f4b2830f
7 changed files with 156 additions and 21 deletions

View file

@ -567,11 +567,13 @@ Important behavior:
identity (`SELECT ... FOR UPDATE`), so a given refresh token is presented to
the provider once across concurrent requests and across two Heimdall
processes sharing one database. Within one process, concurrent refreshes of
one identity share a single attempt and its outcome (success or failure), so
nobody re-presents a token behind a failed holder in that burst. Residual
risk, tracked as follow-up `joystick:follow_up:cross-process-failed-refresh-marker`:
the guard is one presentation per process per burst, so N
Heimdall processes can present one token N times after a transient failure.
one identity share a single attempt and its outcome (success or failure). A
failed or timed-out attempt is recorded on the identity row
(`refresh_failed_at`, written in the same transaction as the lock), and no
process presents that row's refresh token again for 60 seconds: a process
that takes the lock inside the backoff answers the same
`managed_credential_refresh_failed` without calling the provider. A rotation,
a revocation or a re-link clears the marker.
The provider call is bounded at 10 seconds
- a provider whose refresh tokens rotate (Joystick) fails closed: a refresh
response without a new refresh token revokes the credential instead of

View file

@ -16,6 +16,14 @@ export const MANAGED_CREDENTIAL_REFRESH_SKEW_MS = 5 * 60 * 1000;
*/
export const MANAGED_CREDENTIAL_REFRESH_TIMEOUT_MS = 10_000;
/**
* After a refresh attempt fails or times out, no process presents the row's
* refresh token again until this long has passed (the failure is recorded on
* the row, read under the lock). A provider that rotates may have consumed the
* token before the attempt was aborted, and presenting it again reads as theft.
*/
export const MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS = 60 * 1000;
export interface ManagedCredentialContext {
config: HeimdallConfig;
store: HeimdallStore;
@ -68,10 +76,8 @@ function withoutTokens(identity: StoredLinkedIdentity): StoredLinkedIdentity {
* attempt is in flight join it and receive its outcome, success or failure.
* The entry lives exactly as long as the attempt: a later request retries.
*
* Known residual risk (docs/service-contract.md): one presentation per
* process per burst, so N processes may present one token N times. The
* cross-process failed-refresh marker that would close it is
* joystick:follow_up:cross-process-failed-refresh-marker.
* Across processes the row owns it: a failed attempt leaves a marker on the
* row, and a process that takes the lock inside the backoff presents nothing.
*/
const refreshesInFlight = new WeakMap<HeimdallStore, Map<string, Promise<ManagedCredentialOutcome>>>();
@ -120,10 +126,15 @@ async function refreshUnderLock(
const runtime = ctx.oauthRuntimes[provider];
let revocation: ManagedCredentialRevocationReason | undefined;
let failed = false;
const decide = async (locked: StoredLinkedIdentity): Promise<CredentialDecision> => {
if (!locked.accessTokenEncrypted || !dueForRefresh(locked, runtime, clock())) {
return { kind: "keep" };
}
if (locked.refreshFailedAt && clock() - Date.parse(locked.refreshFailedAt) < MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS) {
failed = true;
return { kind: "keep" };
}
const updatedAt = new Date(clock()).toISOString();
const refreshToken = ctx.tokenCustody.decrypt(locked.refreshTokenEncrypted!);
@ -139,7 +150,8 @@ async function refreshUnderLock(
revocation = "provider_revoked";
return { kind: "revoke", updatedAt };
}
throw new ManagedCredentialRefreshFailed(provider);
failed = true;
return { kind: "failed", failedAt: updatedAt };
}
let refreshTokenEncrypted: string;
@ -174,6 +186,9 @@ async function refreshUnderLock(
createdAt: new Date(clock()).toISOString(),
});
}
if (failed) {
throw new ManagedCredentialRefreshFailed(provider);
}
if (!committed) {
return { kind: "revoked", identity: withoutTokens(snapshot) };
}

View file

@ -203,7 +203,13 @@ export class InMemoryStore implements HeimdallStore {
if (decision.kind === "keep") {
return clone(current);
}
if (decision.kind === "failed") {
const marked: StoredLinkedIdentity = { ...current, refreshFailedAt: decision.failedAt };
this.linkedIdentities.set(id, clone(marked));
return clone(marked);
}
const next: StoredLinkedIdentity = { ...current, updatedAt: decision.updatedAt };
delete next.refreshFailedAt;
delete next.accessTokenEncrypted;
delete next.refreshTokenEncrypted;
delete next.tokenExpiresAt;

View file

@ -47,6 +47,7 @@ interface LinkedIdentityRow extends QueryResultRow {
access_token_encrypted: string | null;
refresh_token_encrypted: string | null;
token_expires_at: string | null;
refresh_failed_at: string | null;
scopes: string;
profile_json: Record<string, unknown>;
created_at: string;
@ -207,6 +208,10 @@ function mapLinkedIdentityRow(row: LinkedIdentityRow): StoredLinkedIdentity {
identity.tokenExpiresAt = new Date(row.token_expires_at).toISOString();
}
if (row.refresh_failed_at) {
identity.refreshFailedAt = new Date(row.refresh_failed_at).toISOString();
}
return identity;
}
@ -777,6 +782,7 @@ export class PostgresStore implements HeimdallStore {
access_token_encrypted = EXCLUDED.access_token_encrypted,
refresh_token_encrypted = EXCLUDED.refresh_token_encrypted,
token_expires_at = EXCLUDED.token_expires_at,
refresh_failed_at = NULL,
scopes = EXCLUDED.scopes,
profile_json = EXCLUDED.profile_json,
updated_at = EXCLUDED.updated_at
@ -822,7 +828,13 @@ export class PostgresStore implements HeimdallStore {
const row = mapLinkedIdentityRow(expectRow(locked.rows[0], "updateLinkedIdentityCredential"));
const decision = await decide(row);
let committed = row;
if (decision.kind !== "keep") {
if (decision.kind === "failed") {
const marked = await client.query<LinkedIdentityRow>(
"UPDATE linked_identities SET refresh_failed_at = $2 WHERE id = $1 RETURNING *",
[id, decision.failedAt]
);
committed = mapLinkedIdentityRow(expectRow(marked.rows[0], "updateLinkedIdentityCredential"));
} else if (decision.kind !== "keep") {
const rotated = decision.kind === "rotate" ? decision : undefined;
const written = await client.query<LinkedIdentityRow>(
`
@ -830,6 +842,7 @@ export class PostgresStore implements HeimdallStore {
SET access_token_encrypted = $2,
refresh_token_encrypted = $3,
token_expires_at = $4,
refresh_failed_at = NULL,
scopes = $5,
updated_at = $6
WHERE id = $1

View file

@ -123,6 +123,11 @@ ALTER TABLE auth_completions ADD COLUMN IF NOT EXISTS attempt_id TEXT;
-- compares it against the target the redeeming app states.
ALTER TABLE auth_completions ADD COLUMN IF NOT EXISTS target_id TEXT;
-- When a refresh attempt on this row last failed or timed out, read under the row
-- lock so a second Heimdall process does not present the same refresh token
-- right behind it. Cleared by rotation, revocation and re-link.
ALTER TABLE linked_identities ADD COLUMN IF NOT EXISTS refresh_failed_at TIMESTAMPTZ;
-- One attempt handle may bind at most one unconsumed completion (R21.1): a
-- second completion for the same handle would let the by-attempt UPDATE in
-- consumeAuthCompletionByAttempt match more than one row, silently consuming

View file

@ -33,6 +33,8 @@ export interface StoredLinkedIdentity {
accessTokenEncrypted?: string;
refreshTokenEncrypted?: string;
tokenExpiresAt?: string;
/** When a refresh attempt last failed; cleared by the next rotation, revocation or re-link. */
refreshFailedAt?: string;
scopes: string[];
profileJson: Record<string, unknown>;
createdAt: string;
@ -58,8 +60,9 @@ export interface UpsertLinkedIdentityInput {
/**
* What the owner of a refresh decides after reading the row under its lock.
* `rotate` and `revoke` write token columns, scopes and updated_at only; they
* never touch account, profile or identity fields.
* `rotate` and `revoke` write token columns, scopes, updated_at and clear the
* failed-refresh marker; `failed` writes the marker alone. None touches
* account, profile or identity fields.
*/
export type CredentialDecision =
| { kind: "keep" }
@ -71,7 +74,8 @@ export type CredentialDecision =
scopes: string[];
updatedAt: string;
}
| { kind: "revoke"; updatedAt: string };
| { kind: "revoke"; updatedAt: string }
| { kind: "failed"; failedAt: string };
export interface StoredSession {
id: string;

View file

@ -6,6 +6,7 @@ import { type HeimdallConfig } from "../src/config.js";
import { type Provider } from "../src/contracts.js";
import { createTokenCustody } from "../src/custody.js";
import {
MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS,
MANAGED_CREDENTIAL_REFRESH_SKEW_MS,
ManagedCredentialRefreshFailed,
refreshManagedCredential,
@ -419,17 +420,18 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag
expect(Date.now() - started).toBeLessThan(5_000);
endpoint.mode = "normal";
const outcome = await refreshManagedCredential(ctx, await rig.row());
const afterBackoff = { ...ctx, clock: () => Date.now() + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS };
const outcome = await refreshManagedCredential(afterBackoff, await rig.row());
expect(outcome.kind).toBe("current");
expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1");
});
it("single flight: concurrent refreshes against a never-answering endpoint make one provider call per process, and every caller gets the typed failure", timeout, async () => {
it("single flight: concurrent refreshes against a never-answering endpoint make one provider call across all processes, and every caller gets the typed failure", timeout, async () => {
const rig = await makeRig(provider);
const endpoint = installTokenEndpoint(provider, "refresh-0");
endpoint.mode = "hang";
const snapshot = await rig.row();
const ctxFor = (store: HeimdallStore) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150 });
const ctxFor = (store: HeimdallStore, clock = Date.now) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150, clock });
const results = await Promise.allSettled(
Array.from({ length: 12 }, (_, i) => refreshManagedCredential(ctxFor(rig.stores[i % rig.stores.length]!), snapshot))
@ -439,14 +441,102 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag
for (const result of results) {
expect((result as PromiseRejectedResult).reason).toBeInstanceOf(ManagedCredentialRefreshFailed);
}
expect(endpoint.calls).toBe(rig.stores.length);
// The process that queued behind the failed holder reads the holder's marker under the lock and presents nothing.
expect(endpoint.calls).toBe(1);
expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0");
// The burst is over: a later independent request tries again.
// The burst is over and so is the backoff the failure left on the row: a later independent request tries again.
endpoint.mode = "normal";
const later = await refreshManagedCredential(ctxFor(rig.stores[0]!), await rig.row());
const later = await refreshManagedCredential(ctxFor(rig.stores[0]!, () => Date.now() + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS), await rig.row());
expect(later.kind).toBe("current");
expect(endpoint.calls).toBe(rig.stores.length + 1);
expect(endpoint.calls).toBe(2);
});
it.each(["server_error", "hang"] as const)(
"cross-process failed-refresh marker (%s): the failed attempt is recorded on the row, and another process does not present the token inside the backoff",
timeout,
async (mode) => {
const rig = await makeRig(provider);
const endpoint = installTokenEndpoint(provider, "refresh-0");
endpoint.mode = mode;
const t0 = Date.now();
const ctxFor = (store: HeimdallStore, at: number) => ({
config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 100, clock: () => at,
});
const snapshot = await rig.row();
const first = rig.stores[0]!;
const other = rig.stores[rig.stores.length - 1]!;
await expect(refreshManagedCredential(ctxFor(first, t0), snapshot)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
expect(endpoint.calls).toBe(1);
const marked = await rig.row();
expect(marked.refreshFailedAt).toBe(new Date(t0).toISOString());
expect(marked.accessTokenEncrypted).toBe(snapshot.accessTokenEncrypted);
expect(marked.refreshTokenEncrypted).toBe(snapshot.refreshTokenEncrypted);
expect(marked.tokenExpiresAt).toBe(snapshot.tokenExpiresAt);
// Another process, holding the stale snapshot, one tick before the backoff ends: the locked row decides, and no token call is made.
const backedOff = ctxFor(other, t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS - 1);
await expect(refreshManagedCredential(backedOff, snapshot)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
expect(endpoint.calls).toBe(1);
// Being turned away does not extend the backoff.
expect((await rig.row()).refreshFailedAt).toBe(marked.refreshFailedAt);
expect(await rig.audit()).toEqual([]);
}
);
it("cross-process failed-refresh marker: the backoff ends exactly at its length, a success clears the marker, and a failure after it marks anew", timeout, async () => {
const rig = await makeRig(provider);
const endpoint = installTokenEndpoint(provider, "refresh-0");
endpoint.mode = "server_error";
const t0 = Date.now();
const ctxAt = (at: number) => ({ config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => at });
await expect(refreshManagedCredential(ctxAt(t0), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
const retryAt = t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS;
await expect(refreshManagedCredential(ctxAt(retryAt), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
expect(endpoint.calls).toBe(2);
expect((await rig.row()).refreshFailedAt).toBe(new Date(retryAt).toISOString());
endpoint.mode = "normal";
const laterAt = retryAt + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS;
const outcome = await refreshManagedCredential(ctxAt(laterAt), await rig.row());
expect(outcome.kind).toBe("current");
expect(endpoint.calls).toBe(3);
expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1");
expect((await rig.row()).refreshFailedAt).toBeUndefined();
});
it("cross-process failed-refresh marker: a revocation and a re-link both clear it", timeout, async () => {
const rig = await makeRig(provider);
const endpoint = installTokenEndpoint(provider, "refresh-0");
endpoint.mode = "server_error";
const t0 = Date.now();
const ctxAt = (at: number) => ({ config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => at });
await expect(refreshManagedCredential(ctxAt(t0), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
expect((await rig.row()).refreshFailedAt).toBeDefined();
endpoint.mode = "invalid_grant";
const revoked = await refreshManagedCredential(ctxAt(t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS), await rig.row());
expect(revoked.kind).toBe("revoked");
expect((await rig.row()).refreshFailedAt).toBeUndefined();
// A fresh consent after a failure replaces the whole credential, marker included.
endpoint.mode = "server_error";
const relinkSeed = await rig.row();
await rig.stores[0]!.upsertLinkedIdentity({
accountId: rig.accountId, provider, providerUserId: relinkSeed.providerUserId, scopes: ["identity:read"], profileJson: {},
accessTokenEncrypted: rig.custody.encrypt("access-r"), refreshTokenEncrypted: rig.custody.encrypt("refresh-r"),
tokenExpiresAt: new Date(t0 + 1000).toISOString(), createdAt: relinkSeed.createdAt, updatedAt: new Date(t0).toISOString(),
});
await expect(refreshManagedCredential(ctxAt(t0 + 2000), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
expect((await rig.row()).refreshFailedAt).toBeDefined();
await rig.stores[0]!.upsertLinkedIdentity({
accountId: rig.accountId, provider, providerUserId: relinkSeed.providerUserId, scopes: ["identity:read"], profileJson: {},
accessTokenEncrypted: rig.custody.encrypt("access-s"), refreshTokenEncrypted: rig.custody.encrypt("refresh-s"),
tokenExpiresAt: new Date(t0 + 3000).toISOString(), createdAt: relinkSeed.createdAt, updatedAt: new Date(t0 + 2500).toISOString(),
});
expect((await rig.row()).refreshFailedAt).toBeUndefined();
});
it("a snapshot taken before another caller's refresh does not decide: the locked row does", timeout, async () => {