Record a failed refresh on the linked identity row so another process does not re-present the token
A failed or timed-out refresh attempt now writes refresh_failed_at under the row lock; a process that takes the lock inside a 60 s backoff answers managed_credential_refresh_failed without calling the provider. Rotation, revocation and re-link clear the marker. Closes the cross-process residual that refresh-once-per-token left to a follow-up, and pins it on the two-pool rig. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2bd5096d81
commit
f5f4b2830f
7 changed files with 156 additions and 21 deletions
|
|
@ -567,11 +567,13 @@ Important behavior:
|
|||
identity (`SELECT ... FOR UPDATE`), so a given refresh token is presented to
|
||||
the provider once across concurrent requests and across two Heimdall
|
||||
processes sharing one database. Within one process, concurrent refreshes of
|
||||
one identity share a single attempt and its outcome (success or failure), so
|
||||
nobody re-presents a token behind a failed holder in that burst. Residual
|
||||
risk, tracked as follow-up `joystick:follow_up:cross-process-failed-refresh-marker`:
|
||||
the guard is one presentation per process per burst, so N
|
||||
Heimdall processes can present one token N times after a transient failure.
|
||||
one identity share a single attempt and its outcome (success or failure). A
|
||||
failed or timed-out attempt is recorded on the identity row
|
||||
(`refresh_failed_at`, written in the same transaction as the lock), and no
|
||||
process presents that row's refresh token again for 60 seconds: a process
|
||||
that takes the lock inside the backoff answers the same
|
||||
`managed_credential_refresh_failed` without calling the provider. A rotation,
|
||||
a revocation or a re-link clears the marker.
|
||||
The provider call is bounded at 10 seconds
|
||||
- a provider whose refresh tokens rotate (Joystick) fails closed: a refresh
|
||||
response without a new refresh token revokes the credential instead of
|
||||
|
|
|
|||
|
|
@ -16,6 +16,14 @@ export const MANAGED_CREDENTIAL_REFRESH_SKEW_MS = 5 * 60 * 1000;
|
|||
*/
|
||||
export const MANAGED_CREDENTIAL_REFRESH_TIMEOUT_MS = 10_000;
|
||||
|
||||
/**
|
||||
* After a refresh attempt fails or times out, no process presents the row's
|
||||
* refresh token again until this long has passed (the failure is recorded on
|
||||
* the row, read under the lock). A provider that rotates may have consumed the
|
||||
* token before the attempt was aborted, and presenting it again reads as theft.
|
||||
*/
|
||||
export const MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS = 60 * 1000;
|
||||
|
||||
export interface ManagedCredentialContext {
|
||||
config: HeimdallConfig;
|
||||
store: HeimdallStore;
|
||||
|
|
@ -68,10 +76,8 @@ function withoutTokens(identity: StoredLinkedIdentity): StoredLinkedIdentity {
|
|||
* attempt is in flight join it and receive its outcome, success or failure.
|
||||
* The entry lives exactly as long as the attempt: a later request retries.
|
||||
*
|
||||
* Known residual risk (docs/service-contract.md): one presentation per
|
||||
* process per burst, so N processes may present one token N times. The
|
||||
* cross-process failed-refresh marker that would close it is
|
||||
* joystick:follow_up:cross-process-failed-refresh-marker.
|
||||
* Across processes the row owns it: a failed attempt leaves a marker on the
|
||||
* row, and a process that takes the lock inside the backoff presents nothing.
|
||||
*/
|
||||
const refreshesInFlight = new WeakMap<HeimdallStore, Map<string, Promise<ManagedCredentialOutcome>>>();
|
||||
|
||||
|
|
@ -120,10 +126,15 @@ async function refreshUnderLock(
|
|||
const runtime = ctx.oauthRuntimes[provider];
|
||||
|
||||
let revocation: ManagedCredentialRevocationReason | undefined;
|
||||
let failed = false;
|
||||
const decide = async (locked: StoredLinkedIdentity): Promise<CredentialDecision> => {
|
||||
if (!locked.accessTokenEncrypted || !dueForRefresh(locked, runtime, clock())) {
|
||||
return { kind: "keep" };
|
||||
}
|
||||
if (locked.refreshFailedAt && clock() - Date.parse(locked.refreshFailedAt) < MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS) {
|
||||
failed = true;
|
||||
return { kind: "keep" };
|
||||
}
|
||||
const updatedAt = new Date(clock()).toISOString();
|
||||
const refreshToken = ctx.tokenCustody.decrypt(locked.refreshTokenEncrypted!);
|
||||
|
||||
|
|
@ -139,7 +150,8 @@ async function refreshUnderLock(
|
|||
revocation = "provider_revoked";
|
||||
return { kind: "revoke", updatedAt };
|
||||
}
|
||||
throw new ManagedCredentialRefreshFailed(provider);
|
||||
failed = true;
|
||||
return { kind: "failed", failedAt: updatedAt };
|
||||
}
|
||||
|
||||
let refreshTokenEncrypted: string;
|
||||
|
|
@ -174,6 +186,9 @@ async function refreshUnderLock(
|
|||
createdAt: new Date(clock()).toISOString(),
|
||||
});
|
||||
}
|
||||
if (failed) {
|
||||
throw new ManagedCredentialRefreshFailed(provider);
|
||||
}
|
||||
if (!committed) {
|
||||
return { kind: "revoked", identity: withoutTokens(snapshot) };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -203,7 +203,13 @@ export class InMemoryStore implements HeimdallStore {
|
|||
if (decision.kind === "keep") {
|
||||
return clone(current);
|
||||
}
|
||||
if (decision.kind === "failed") {
|
||||
const marked: StoredLinkedIdentity = { ...current, refreshFailedAt: decision.failedAt };
|
||||
this.linkedIdentities.set(id, clone(marked));
|
||||
return clone(marked);
|
||||
}
|
||||
const next: StoredLinkedIdentity = { ...current, updatedAt: decision.updatedAt };
|
||||
delete next.refreshFailedAt;
|
||||
delete next.accessTokenEncrypted;
|
||||
delete next.refreshTokenEncrypted;
|
||||
delete next.tokenExpiresAt;
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ interface LinkedIdentityRow extends QueryResultRow {
|
|||
access_token_encrypted: string | null;
|
||||
refresh_token_encrypted: string | null;
|
||||
token_expires_at: string | null;
|
||||
refresh_failed_at: string | null;
|
||||
scopes: string;
|
||||
profile_json: Record<string, unknown>;
|
||||
created_at: string;
|
||||
|
|
@ -207,6 +208,10 @@ function mapLinkedIdentityRow(row: LinkedIdentityRow): StoredLinkedIdentity {
|
|||
identity.tokenExpiresAt = new Date(row.token_expires_at).toISOString();
|
||||
}
|
||||
|
||||
if (row.refresh_failed_at) {
|
||||
identity.refreshFailedAt = new Date(row.refresh_failed_at).toISOString();
|
||||
}
|
||||
|
||||
return identity;
|
||||
}
|
||||
|
||||
|
|
@ -777,6 +782,7 @@ export class PostgresStore implements HeimdallStore {
|
|||
access_token_encrypted = EXCLUDED.access_token_encrypted,
|
||||
refresh_token_encrypted = EXCLUDED.refresh_token_encrypted,
|
||||
token_expires_at = EXCLUDED.token_expires_at,
|
||||
refresh_failed_at = NULL,
|
||||
scopes = EXCLUDED.scopes,
|
||||
profile_json = EXCLUDED.profile_json,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
|
|
@ -822,7 +828,13 @@ export class PostgresStore implements HeimdallStore {
|
|||
const row = mapLinkedIdentityRow(expectRow(locked.rows[0], "updateLinkedIdentityCredential"));
|
||||
const decision = await decide(row);
|
||||
let committed = row;
|
||||
if (decision.kind !== "keep") {
|
||||
if (decision.kind === "failed") {
|
||||
const marked = await client.query<LinkedIdentityRow>(
|
||||
"UPDATE linked_identities SET refresh_failed_at = $2 WHERE id = $1 RETURNING *",
|
||||
[id, decision.failedAt]
|
||||
);
|
||||
committed = mapLinkedIdentityRow(expectRow(marked.rows[0], "updateLinkedIdentityCredential"));
|
||||
} else if (decision.kind !== "keep") {
|
||||
const rotated = decision.kind === "rotate" ? decision : undefined;
|
||||
const written = await client.query<LinkedIdentityRow>(
|
||||
`
|
||||
|
|
@ -830,6 +842,7 @@ export class PostgresStore implements HeimdallStore {
|
|||
SET access_token_encrypted = $2,
|
||||
refresh_token_encrypted = $3,
|
||||
token_expires_at = $4,
|
||||
refresh_failed_at = NULL,
|
||||
scopes = $5,
|
||||
updated_at = $6
|
||||
WHERE id = $1
|
||||
|
|
|
|||
|
|
@ -123,6 +123,11 @@ ALTER TABLE auth_completions ADD COLUMN IF NOT EXISTS attempt_id TEXT;
|
|||
-- compares it against the target the redeeming app states.
|
||||
ALTER TABLE auth_completions ADD COLUMN IF NOT EXISTS target_id TEXT;
|
||||
|
||||
-- When a refresh attempt on this row last failed or timed out, read under the row
|
||||
-- lock so a second Heimdall process does not present the same refresh token
|
||||
-- right behind it. Cleared by rotation, revocation and re-link.
|
||||
ALTER TABLE linked_identities ADD COLUMN IF NOT EXISTS refresh_failed_at TIMESTAMPTZ;
|
||||
|
||||
-- One attempt handle may bind at most one unconsumed completion (R21.1): a
|
||||
-- second completion for the same handle would let the by-attempt UPDATE in
|
||||
-- consumeAuthCompletionByAttempt match more than one row, silently consuming
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ export interface StoredLinkedIdentity {
|
|||
accessTokenEncrypted?: string;
|
||||
refreshTokenEncrypted?: string;
|
||||
tokenExpiresAt?: string;
|
||||
/** When a refresh attempt last failed; cleared by the next rotation, revocation or re-link. */
|
||||
refreshFailedAt?: string;
|
||||
scopes: string[];
|
||||
profileJson: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
|
|
@ -58,8 +60,9 @@ export interface UpsertLinkedIdentityInput {
|
|||
|
||||
/**
|
||||
* What the owner of a refresh decides after reading the row under its lock.
|
||||
* `rotate` and `revoke` write token columns, scopes and updated_at only; they
|
||||
* never touch account, profile or identity fields.
|
||||
* `rotate` and `revoke` write token columns, scopes, updated_at and clear the
|
||||
* failed-refresh marker; `failed` writes the marker alone. None touches
|
||||
* account, profile or identity fields.
|
||||
*/
|
||||
export type CredentialDecision =
|
||||
| { kind: "keep" }
|
||||
|
|
@ -71,7 +74,8 @@ export type CredentialDecision =
|
|||
scopes: string[];
|
||||
updatedAt: string;
|
||||
}
|
||||
| { kind: "revoke"; updatedAt: string };
|
||||
| { kind: "revoke"; updatedAt: string }
|
||||
| { kind: "failed"; failedAt: string };
|
||||
|
||||
export interface StoredSession {
|
||||
id: string;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { type HeimdallConfig } from "../src/config.js";
|
|||
import { type Provider } from "../src/contracts.js";
|
||||
import { createTokenCustody } from "../src/custody.js";
|
||||
import {
|
||||
MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS,
|
||||
MANAGED_CREDENTIAL_REFRESH_SKEW_MS,
|
||||
ManagedCredentialRefreshFailed,
|
||||
refreshManagedCredential,
|
||||
|
|
@ -419,17 +420,18 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag
|
|||
expect(Date.now() - started).toBeLessThan(5_000);
|
||||
|
||||
endpoint.mode = "normal";
|
||||
const outcome = await refreshManagedCredential(ctx, await rig.row());
|
||||
const afterBackoff = { ...ctx, clock: () => Date.now() + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS };
|
||||
const outcome = await refreshManagedCredential(afterBackoff, await rig.row());
|
||||
expect(outcome.kind).toBe("current");
|
||||
expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1");
|
||||
});
|
||||
|
||||
it("single flight: concurrent refreshes against a never-answering endpoint make one provider call per process, and every caller gets the typed failure", timeout, async () => {
|
||||
it("single flight: concurrent refreshes against a never-answering endpoint make one provider call across all processes, and every caller gets the typed failure", timeout, async () => {
|
||||
const rig = await makeRig(provider);
|
||||
const endpoint = installTokenEndpoint(provider, "refresh-0");
|
||||
endpoint.mode = "hang";
|
||||
const snapshot = await rig.row();
|
||||
const ctxFor = (store: HeimdallStore) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150 });
|
||||
const ctxFor = (store: HeimdallStore, clock = Date.now) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150, clock });
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
Array.from({ length: 12 }, (_, i) => refreshManagedCredential(ctxFor(rig.stores[i % rig.stores.length]!), snapshot))
|
||||
|
|
@ -439,14 +441,102 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag
|
|||
for (const result of results) {
|
||||
expect((result as PromiseRejectedResult).reason).toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
}
|
||||
expect(endpoint.calls).toBe(rig.stores.length);
|
||||
// The process that queued behind the failed holder reads the holder's marker under the lock and presents nothing.
|
||||
expect(endpoint.calls).toBe(1);
|
||||
expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0");
|
||||
|
||||
// The burst is over: a later independent request tries again.
|
||||
// The burst is over and so is the backoff the failure left on the row: a later independent request tries again.
|
||||
endpoint.mode = "normal";
|
||||
const later = await refreshManagedCredential(ctxFor(rig.stores[0]!), await rig.row());
|
||||
const later = await refreshManagedCredential(ctxFor(rig.stores[0]!, () => Date.now() + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS), await rig.row());
|
||||
expect(later.kind).toBe("current");
|
||||
expect(endpoint.calls).toBe(rig.stores.length + 1);
|
||||
expect(endpoint.calls).toBe(2);
|
||||
});
|
||||
|
||||
it.each(["server_error", "hang"] as const)(
|
||||
"cross-process failed-refresh marker (%s): the failed attempt is recorded on the row, and another process does not present the token inside the backoff",
|
||||
timeout,
|
||||
async (mode) => {
|
||||
const rig = await makeRig(provider);
|
||||
const endpoint = installTokenEndpoint(provider, "refresh-0");
|
||||
endpoint.mode = mode;
|
||||
const t0 = Date.now();
|
||||
const ctxFor = (store: HeimdallStore, at: number) => ({
|
||||
config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 100, clock: () => at,
|
||||
});
|
||||
const snapshot = await rig.row();
|
||||
const first = rig.stores[0]!;
|
||||
const other = rig.stores[rig.stores.length - 1]!;
|
||||
|
||||
await expect(refreshManagedCredential(ctxFor(first, t0), snapshot)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
expect(endpoint.calls).toBe(1);
|
||||
const marked = await rig.row();
|
||||
expect(marked.refreshFailedAt).toBe(new Date(t0).toISOString());
|
||||
expect(marked.accessTokenEncrypted).toBe(snapshot.accessTokenEncrypted);
|
||||
expect(marked.refreshTokenEncrypted).toBe(snapshot.refreshTokenEncrypted);
|
||||
expect(marked.tokenExpiresAt).toBe(snapshot.tokenExpiresAt);
|
||||
|
||||
// Another process, holding the stale snapshot, one tick before the backoff ends: the locked row decides, and no token call is made.
|
||||
const backedOff = ctxFor(other, t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS - 1);
|
||||
await expect(refreshManagedCredential(backedOff, snapshot)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
expect(endpoint.calls).toBe(1);
|
||||
// Being turned away does not extend the backoff.
|
||||
expect((await rig.row()).refreshFailedAt).toBe(marked.refreshFailedAt);
|
||||
expect(await rig.audit()).toEqual([]);
|
||||
}
|
||||
);
|
||||
|
||||
it("cross-process failed-refresh marker: the backoff ends exactly at its length, a success clears the marker, and a failure after it marks anew", timeout, async () => {
|
||||
const rig = await makeRig(provider);
|
||||
const endpoint = installTokenEndpoint(provider, "refresh-0");
|
||||
endpoint.mode = "server_error";
|
||||
const t0 = Date.now();
|
||||
const ctxAt = (at: number) => ({ config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => at });
|
||||
await expect(refreshManagedCredential(ctxAt(t0), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
|
||||
const retryAt = t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS;
|
||||
await expect(refreshManagedCredential(ctxAt(retryAt), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
expect(endpoint.calls).toBe(2);
|
||||
expect((await rig.row()).refreshFailedAt).toBe(new Date(retryAt).toISOString());
|
||||
|
||||
endpoint.mode = "normal";
|
||||
const laterAt = retryAt + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS;
|
||||
const outcome = await refreshManagedCredential(ctxAt(laterAt), await rig.row());
|
||||
expect(outcome.kind).toBe("current");
|
||||
expect(endpoint.calls).toBe(3);
|
||||
expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1");
|
||||
expect((await rig.row()).refreshFailedAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("cross-process failed-refresh marker: a revocation and a re-link both clear it", timeout, async () => {
|
||||
const rig = await makeRig(provider);
|
||||
const endpoint = installTokenEndpoint(provider, "refresh-0");
|
||||
endpoint.mode = "server_error";
|
||||
const t0 = Date.now();
|
||||
const ctxAt = (at: number) => ({ config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => at });
|
||||
await expect(refreshManagedCredential(ctxAt(t0), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
expect((await rig.row()).refreshFailedAt).toBeDefined();
|
||||
|
||||
endpoint.mode = "invalid_grant";
|
||||
const revoked = await refreshManagedCredential(ctxAt(t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS), await rig.row());
|
||||
expect(revoked.kind).toBe("revoked");
|
||||
expect((await rig.row()).refreshFailedAt).toBeUndefined();
|
||||
|
||||
// A fresh consent after a failure replaces the whole credential, marker included.
|
||||
endpoint.mode = "server_error";
|
||||
const relinkSeed = await rig.row();
|
||||
await rig.stores[0]!.upsertLinkedIdentity({
|
||||
accountId: rig.accountId, provider, providerUserId: relinkSeed.providerUserId, scopes: ["identity:read"], profileJson: {},
|
||||
accessTokenEncrypted: rig.custody.encrypt("access-r"), refreshTokenEncrypted: rig.custody.encrypt("refresh-r"),
|
||||
tokenExpiresAt: new Date(t0 + 1000).toISOString(), createdAt: relinkSeed.createdAt, updatedAt: new Date(t0).toISOString(),
|
||||
});
|
||||
await expect(refreshManagedCredential(ctxAt(t0 + 2000), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed);
|
||||
expect((await rig.row()).refreshFailedAt).toBeDefined();
|
||||
await rig.stores[0]!.upsertLinkedIdentity({
|
||||
accountId: rig.accountId, provider, providerUserId: relinkSeed.providerUserId, scopes: ["identity:read"], profileJson: {},
|
||||
accessTokenEncrypted: rig.custody.encrypt("access-s"), refreshTokenEncrypted: rig.custody.encrypt("refresh-s"),
|
||||
tokenExpiresAt: new Date(t0 + 3000).toISOString(), createdAt: relinkSeed.createdAt, updatedAt: new Date(t0 + 2500).toISOString(),
|
||||
});
|
||||
expect((await rig.row()).refreshFailedAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("a snapshot taken before another caller's refresh does not decide: the locked row does", timeout, async () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue