diff --git a/docs/service-contract.md b/docs/service-contract.md index e6ad6c1..f27e5e1 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -567,11 +567,13 @@ Important behavior: identity (`SELECT ... FOR UPDATE`), so a given refresh token is presented to the provider once across concurrent requests and across two Heimdall processes sharing one database. Within one process, concurrent refreshes of - one identity share a single attempt and its outcome (success or failure), so - nobody re-presents a token behind a failed holder in that burst. Residual - risk, tracked as follow-up `joystick:follow_up:cross-process-failed-refresh-marker`: - the guard is one presentation per process per burst, so N - Heimdall processes can present one token N times after a transient failure. + one identity share a single attempt and its outcome (success or failure). A + failed or timed-out attempt is recorded on the identity row + (`refresh_failed_at`, written in the same transaction as the lock), and no + process presents that row's refresh token again for 60 seconds: a process + that takes the lock inside the backoff answers the same + `managed_credential_refresh_failed` without calling the provider. A rotation, + a revocation or a re-link clears the marker. The provider call is bounded at 10 seconds - a provider whose refresh tokens rotate (Joystick) fails closed: a refresh response without a new refresh token revokes the credential instead of diff --git a/src/managed-credential.ts b/src/managed-credential.ts index 32e9f51..063b733 100644 --- a/src/managed-credential.ts +++ b/src/managed-credential.ts @@ -16,6 +16,14 @@ export const MANAGED_CREDENTIAL_REFRESH_SKEW_MS = 5 * 60 * 1000; */ export const MANAGED_CREDENTIAL_REFRESH_TIMEOUT_MS = 10_000; +/** + * After a refresh attempt fails or times out, no process presents the row's + * refresh token again until this long has passed (the failure is recorded on + * the row, read under the lock). A provider that rotates may have consumed the + * token before the attempt was aborted, and presenting it again reads as theft. + */ +export const MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS = 60 * 1000; + export interface ManagedCredentialContext { config: HeimdallConfig; store: HeimdallStore; @@ -68,10 +76,8 @@ function withoutTokens(identity: StoredLinkedIdentity): StoredLinkedIdentity { * attempt is in flight join it and receive its outcome, success or failure. * The entry lives exactly as long as the attempt: a later request retries. * - * Known residual risk (docs/service-contract.md): one presentation per - * process per burst, so N processes may present one token N times. The - * cross-process failed-refresh marker that would close it is - * joystick:follow_up:cross-process-failed-refresh-marker. + * Across processes the row owns it: a failed attempt leaves a marker on the + * row, and a process that takes the lock inside the backoff presents nothing. */ const refreshesInFlight = new WeakMap>>(); @@ -120,10 +126,15 @@ async function refreshUnderLock( const runtime = ctx.oauthRuntimes[provider]; let revocation: ManagedCredentialRevocationReason | undefined; + let failed = false; const decide = async (locked: StoredLinkedIdentity): Promise => { if (!locked.accessTokenEncrypted || !dueForRefresh(locked, runtime, clock())) { return { kind: "keep" }; } + if (locked.refreshFailedAt && clock() - Date.parse(locked.refreshFailedAt) < MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS) { + failed = true; + return { kind: "keep" }; + } const updatedAt = new Date(clock()).toISOString(); const refreshToken = ctx.tokenCustody.decrypt(locked.refreshTokenEncrypted!); @@ -139,7 +150,8 @@ async function refreshUnderLock( revocation = "provider_revoked"; return { kind: "revoke", updatedAt }; } - throw new ManagedCredentialRefreshFailed(provider); + failed = true; + return { kind: "failed", failedAt: updatedAt }; } let refreshTokenEncrypted: string; @@ -174,6 +186,9 @@ async function refreshUnderLock( createdAt: new Date(clock()).toISOString(), }); } + if (failed) { + throw new ManagedCredentialRefreshFailed(provider); + } if (!committed) { return { kind: "revoked", identity: withoutTokens(snapshot) }; } diff --git a/src/store/in-memory.ts b/src/store/in-memory.ts index e829900..e86490d 100644 --- a/src/store/in-memory.ts +++ b/src/store/in-memory.ts @@ -203,7 +203,13 @@ export class InMemoryStore implements HeimdallStore { if (decision.kind === "keep") { return clone(current); } + if (decision.kind === "failed") { + const marked: StoredLinkedIdentity = { ...current, refreshFailedAt: decision.failedAt }; + this.linkedIdentities.set(id, clone(marked)); + return clone(marked); + } const next: StoredLinkedIdentity = { ...current, updatedAt: decision.updatedAt }; + delete next.refreshFailedAt; delete next.accessTokenEncrypted; delete next.refreshTokenEncrypted; delete next.tokenExpiresAt; diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 3d34047..0c4fc05 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -47,6 +47,7 @@ interface LinkedIdentityRow extends QueryResultRow { access_token_encrypted: string | null; refresh_token_encrypted: string | null; token_expires_at: string | null; + refresh_failed_at: string | null; scopes: string; profile_json: Record; created_at: string; @@ -207,6 +208,10 @@ function mapLinkedIdentityRow(row: LinkedIdentityRow): StoredLinkedIdentity { identity.tokenExpiresAt = new Date(row.token_expires_at).toISOString(); } + if (row.refresh_failed_at) { + identity.refreshFailedAt = new Date(row.refresh_failed_at).toISOString(); + } + return identity; } @@ -777,6 +782,7 @@ export class PostgresStore implements HeimdallStore { access_token_encrypted = EXCLUDED.access_token_encrypted, refresh_token_encrypted = EXCLUDED.refresh_token_encrypted, token_expires_at = EXCLUDED.token_expires_at, + refresh_failed_at = NULL, scopes = EXCLUDED.scopes, profile_json = EXCLUDED.profile_json, updated_at = EXCLUDED.updated_at @@ -822,7 +828,13 @@ export class PostgresStore implements HeimdallStore { const row = mapLinkedIdentityRow(expectRow(locked.rows[0], "updateLinkedIdentityCredential")); const decision = await decide(row); let committed = row; - if (decision.kind !== "keep") { + if (decision.kind === "failed") { + const marked = await client.query( + "UPDATE linked_identities SET refresh_failed_at = $2 WHERE id = $1 RETURNING *", + [id, decision.failedAt] + ); + committed = mapLinkedIdentityRow(expectRow(marked.rows[0], "updateLinkedIdentityCredential")); + } else if (decision.kind !== "keep") { const rotated = decision.kind === "rotate" ? decision : undefined; const written = await client.query( ` @@ -830,6 +842,7 @@ export class PostgresStore implements HeimdallStore { SET access_token_encrypted = $2, refresh_token_encrypted = $3, token_expires_at = $4, + refresh_failed_at = NULL, scopes = $5, updated_at = $6 WHERE id = $1 diff --git a/src/store/schema.ts b/src/store/schema.ts index 294ee2d..7aa6823 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -123,6 +123,11 @@ ALTER TABLE auth_completions ADD COLUMN IF NOT EXISTS attempt_id TEXT; -- compares it against the target the redeeming app states. ALTER TABLE auth_completions ADD COLUMN IF NOT EXISTS target_id TEXT; +-- When a refresh attempt on this row last failed or timed out, read under the row +-- lock so a second Heimdall process does not present the same refresh token +-- right behind it. Cleared by rotation, revocation and re-link. +ALTER TABLE linked_identities ADD COLUMN IF NOT EXISTS refresh_failed_at TIMESTAMPTZ; + -- One attempt handle may bind at most one unconsumed completion (R21.1): a -- second completion for the same handle would let the by-attempt UPDATE in -- consumeAuthCompletionByAttempt match more than one row, silently consuming diff --git a/src/store/types.ts b/src/store/types.ts index bb2e080..c795247 100644 --- a/src/store/types.ts +++ b/src/store/types.ts @@ -33,6 +33,8 @@ export interface StoredLinkedIdentity { accessTokenEncrypted?: string; refreshTokenEncrypted?: string; tokenExpiresAt?: string; + /** When a refresh attempt last failed; cleared by the next rotation, revocation or re-link. */ + refreshFailedAt?: string; scopes: string[]; profileJson: Record; createdAt: string; @@ -58,8 +60,9 @@ export interface UpsertLinkedIdentityInput { /** * What the owner of a refresh decides after reading the row under its lock. - * `rotate` and `revoke` write token columns, scopes and updated_at only; they - * never touch account, profile or identity fields. + * `rotate` and `revoke` write token columns, scopes, updated_at and clear the + * failed-refresh marker; `failed` writes the marker alone. None touches + * account, profile or identity fields. */ export type CredentialDecision = | { kind: "keep" } @@ -71,7 +74,8 @@ export type CredentialDecision = scopes: string[]; updatedAt: string; } - | { kind: "revoke"; updatedAt: string }; + | { kind: "revoke"; updatedAt: string } + | { kind: "failed"; failedAt: string }; export interface StoredSession { id: string; diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index ffe1472..03bcc1a 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -6,6 +6,7 @@ import { type HeimdallConfig } from "../src/config.js"; import { type Provider } from "../src/contracts.js"; import { createTokenCustody } from "../src/custody.js"; import { + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS, MANAGED_CREDENTIAL_REFRESH_SKEW_MS, ManagedCredentialRefreshFailed, refreshManagedCredential, @@ -419,17 +420,18 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag expect(Date.now() - started).toBeLessThan(5_000); endpoint.mode = "normal"; - const outcome = await refreshManagedCredential(ctx, await rig.row()); + const afterBackoff = { ...ctx, clock: () => Date.now() + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS }; + const outcome = await refreshManagedCredential(afterBackoff, await rig.row()); expect(outcome.kind).toBe("current"); expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1"); }); - it("single flight: concurrent refreshes against a never-answering endpoint make one provider call per process, and every caller gets the typed failure", timeout, async () => { + it("single flight: concurrent refreshes against a never-answering endpoint make one provider call across all processes, and every caller gets the typed failure", timeout, async () => { const rig = await makeRig(provider); const endpoint = installTokenEndpoint(provider, "refresh-0"); endpoint.mode = "hang"; const snapshot = await rig.row(); - const ctxFor = (store: HeimdallStore) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150 }); + const ctxFor = (store: HeimdallStore, clock = Date.now) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150, clock }); const results = await Promise.allSettled( Array.from({ length: 12 }, (_, i) => refreshManagedCredential(ctxFor(rig.stores[i % rig.stores.length]!), snapshot)) @@ -439,14 +441,102 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag for (const result of results) { expect((result as PromiseRejectedResult).reason).toBeInstanceOf(ManagedCredentialRefreshFailed); } - expect(endpoint.calls).toBe(rig.stores.length); + // The process that queued behind the failed holder reads the holder's marker under the lock and presents nothing. + expect(endpoint.calls).toBe(1); expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0"); - // The burst is over: a later independent request tries again. + // The burst is over and so is the backoff the failure left on the row: a later independent request tries again. endpoint.mode = "normal"; - const later = await refreshManagedCredential(ctxFor(rig.stores[0]!), await rig.row()); + const later = await refreshManagedCredential(ctxFor(rig.stores[0]!, () => Date.now() + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS), await rig.row()); expect(later.kind).toBe("current"); - expect(endpoint.calls).toBe(rig.stores.length + 1); + expect(endpoint.calls).toBe(2); + }); + + it.each(["server_error", "hang"] as const)( + "cross-process failed-refresh marker (%s): the failed attempt is recorded on the row, and another process does not present the token inside the backoff", + timeout, + async (mode) => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = mode; + const t0 = Date.now(); + const ctxFor = (store: HeimdallStore, at: number) => ({ + config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 100, clock: () => at, + }); + const snapshot = await rig.row(); + const first = rig.stores[0]!; + const other = rig.stores[rig.stores.length - 1]!; + + await expect(refreshManagedCredential(ctxFor(first, t0), snapshot)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect(endpoint.calls).toBe(1); + const marked = await rig.row(); + expect(marked.refreshFailedAt).toBe(new Date(t0).toISOString()); + expect(marked.accessTokenEncrypted).toBe(snapshot.accessTokenEncrypted); + expect(marked.refreshTokenEncrypted).toBe(snapshot.refreshTokenEncrypted); + expect(marked.tokenExpiresAt).toBe(snapshot.tokenExpiresAt); + + // Another process, holding the stale snapshot, one tick before the backoff ends: the locked row decides, and no token call is made. + const backedOff = ctxFor(other, t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS - 1); + await expect(refreshManagedCredential(backedOff, snapshot)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect(endpoint.calls).toBe(1); + // Being turned away does not extend the backoff. + expect((await rig.row()).refreshFailedAt).toBe(marked.refreshFailedAt); + expect(await rig.audit()).toEqual([]); + } + ); + + it("cross-process failed-refresh marker: the backoff ends exactly at its length, a success clears the marker, and a failure after it marks anew", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "server_error"; + const t0 = Date.now(); + const ctxAt = (at: number) => ({ config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => at }); + await expect(refreshManagedCredential(ctxAt(t0), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + + const retryAt = t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS; + await expect(refreshManagedCredential(ctxAt(retryAt), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect(endpoint.calls).toBe(2); + expect((await rig.row()).refreshFailedAt).toBe(new Date(retryAt).toISOString()); + + endpoint.mode = "normal"; + const laterAt = retryAt + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS; + const outcome = await refreshManagedCredential(ctxAt(laterAt), await rig.row()); + expect(outcome.kind).toBe("current"); + expect(endpoint.calls).toBe(3); + expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1"); + expect((await rig.row()).refreshFailedAt).toBeUndefined(); + }); + + it("cross-process failed-refresh marker: a revocation and a re-link both clear it", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "server_error"; + const t0 = Date.now(); + const ctxAt = (at: number) => ({ config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => at }); + await expect(refreshManagedCredential(ctxAt(t0), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect((await rig.row()).refreshFailedAt).toBeDefined(); + + endpoint.mode = "invalid_grant"; + const revoked = await refreshManagedCredential(ctxAt(t0 + MANAGED_CREDENTIAL_REFRESH_BACKOFF_MS), await rig.row()); + expect(revoked.kind).toBe("revoked"); + expect((await rig.row()).refreshFailedAt).toBeUndefined(); + + // A fresh consent after a failure replaces the whole credential, marker included. + endpoint.mode = "server_error"; + const relinkSeed = await rig.row(); + await rig.stores[0]!.upsertLinkedIdentity({ + accountId: rig.accountId, provider, providerUserId: relinkSeed.providerUserId, scopes: ["identity:read"], profileJson: {}, + accessTokenEncrypted: rig.custody.encrypt("access-r"), refreshTokenEncrypted: rig.custody.encrypt("refresh-r"), + tokenExpiresAt: new Date(t0 + 1000).toISOString(), createdAt: relinkSeed.createdAt, updatedAt: new Date(t0).toISOString(), + }); + await expect(refreshManagedCredential(ctxAt(t0 + 2000), await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect((await rig.row()).refreshFailedAt).toBeDefined(); + await rig.stores[0]!.upsertLinkedIdentity({ + accountId: rig.accountId, provider, providerUserId: relinkSeed.providerUserId, scopes: ["identity:read"], profileJson: {}, + accessTokenEncrypted: rig.custody.encrypt("access-s"), refreshTokenEncrypted: rig.custody.encrypt("refresh-s"), + tokenExpiresAt: new Date(t0 + 3000).toISOString(), createdAt: relinkSeed.createdAt, updatedAt: new Date(t0 + 2500).toISOString(), + }); + expect((await rig.row()).refreshFailedAt).toBeUndefined(); }); it("a snapshot taken before another caller's refresh does not decide: the locked row does", timeout, async () => {