From d2d0778847546bab1fb1be0b5653f93b90cb25cd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:26:56 +0200 Subject: [PATCH 01/85] Delete the inline signing-key PEM input GC_ACCESS_SIGNING_PRIVATE_KEY_PEM put the signing key's bytes into the process environment, and so into any binding, unit or plan that carries environment. The recipe never declared it and GC_ACCESS_SIGNING_PRIVATE_KEY_PATH already covers the key as a file, which a binding binds through [workload.secret_files]. The signing key is now always a file. Idunn secret-files cut, Cut 1 (delete first). Co-Authored-By: Claude Opus 5.5 --- .env.example | 1 - docs/architecture.md | 8 +++----- docs/service-contract.md | 3 +-- src/config.ts | 6 ------ src/signing.ts | 9 ++------- 5 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 9b1ba8b..8d91415 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,6 @@ GC_ACCESS_APPLY_SCHEMA_ON_STARTUP=1 GC_ACCESS_SESSION_TTL_SECONDS=3600 GC_ACCESS_STATE_TTL_SECONDS=600 GC_ACCESS_COMPLETION_TTL_SECONDS=300 -GC_ACCESS_SIGNING_PRIVATE_KEY_PEM= GC_ACCESS_SIGNING_PRIVATE_KEY_PATH= GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=0 GC_ACCESS_SIGNING_KEY_ID= diff --git a/docs/architecture.md b/docs/architecture.md index 5f290af..ed1abc4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -578,7 +578,6 @@ Suggested generic env surface: - `GC_ACCESS_BASE_URL=https://heimdall.gamecult.org` - `GC_ACCESS_INTERNAL_URL=http://127.0.0.1:4100` - `GC_ACCESS_SESSION_SECRET=...` -- `GC_ACCESS_SIGNING_PRIVATE_KEY_PEM=...` - `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH=/var/lib/heimdall/signing-key.pem` - `GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1` - `GC_ACCESS_SIGNING_KEY_ID=...` @@ -604,10 +603,9 @@ Suggested generic env surface: Only configure the providers Heimdall actually owns for a given deployment. -If `GC_ACCESS_SIGNING_PRIVATE_KEY_PEM` is absent, the current skeleton falls -back to `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH` if present, and can bootstrap that -file on first boot when `GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1`. If -neither PEM nor path is configured, the current skeleton falls back to an +The signing key is always a file: `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH` names +it, and Heimdall can bootstrap that file on first boot when +`GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1`. If no path is configured, the current skeleton falls back to an ephemeral dev signing key. That is acceptable for local iteration and absolutely not acceptable for real deployment. diff --git a/docs/service-contract.md b/docs/service-contract.md index 54df1a9..68eb6a5 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -44,12 +44,11 @@ Purpose: Notes: - current skeleton signs with Ed25519 -- `GC_ACCESS_SIGNING_PRIVATE_KEY_PEM` still works for direct secret injection - `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH` lets the service load a persisted key from disk - if the path is missing and `GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1`, the service bootstraps a new key file on first start -- if neither PEM nor path is configured, the service still generates an +- if no path is configured, the service still generates an ephemeral dev key at startup and deserves no sympathy when the `kid` changes ### `GET /.well-known/heimdall-configuration` diff --git a/src/config.ts b/src/config.ts index c60b96a..58e5269 100644 --- a/src/config.ts +++ b/src/config.ts @@ -35,7 +35,6 @@ export interface HeimdallConfig { refreshTtlSeconds: number; stateTtlSeconds: number; completionTtlSeconds: number; - signingPrivateKeyPem?: string; signingPrivateKeyPath?: string; bootstrapSigningPrivateKeyOnMissing: boolean; signingKeyId?: string; @@ -278,11 +277,6 @@ export function loadConfig( config.odinCultMeshUri = odinCultMeshUri; } - const signingPrivateKeyPem = env.GC_ACCESS_SIGNING_PRIVATE_KEY_PEM?.replace(/\\n/g, "\n"); - if (signingPrivateKeyPem) { - config.signingPrivateKeyPem = signingPrivateKeyPem; - } - if (env.GC_ACCESS_SIGNING_PRIVATE_KEY_PATH) { config.signingPrivateKeyPath = env.GC_ACCESS_SIGNING_PRIVATE_KEY_PATH; } diff --git a/src/signing.ts b/src/signing.ts index 72a5728..1a41a97 100644 --- a/src/signing.ts +++ b/src/signing.ts @@ -26,7 +26,7 @@ export interface RuntimeKeyMaterial { privateKey: KeyObject; publicKey: KeyObject; publicJwk: PublicSigningJwk; - source: "configured_pem" | "configured_file" | "bootstrapped_file" | "ephemeral_dev"; + source: "configured_file" | "bootstrapped_file" | "ephemeral_dev"; } function encodeBase64Url(input: string | Buffer): string { @@ -110,12 +110,7 @@ function createOrLoadPrivateKeyFromPath(config: HeimdallConfig): { } export function createRuntimeKeyMaterial(config: HeimdallConfig): RuntimeKeyMaterial { - const loaded = config.signingPrivateKeyPem - ? { - privateKey: createPrivateKey(config.signingPrivateKeyPem), - source: "configured_pem" as const, - } - : createOrLoadPrivateKeyFromPath(config); + const loaded = createOrLoadPrivateKeyFromPath(config); const privateKey = loaded.privateKey; const publicKey = createPublicKey(privateKey); const exportedJwk = publicKey.export({ format: "jwk" }) as Jwk; From ac863529724bc0f9ae897d4db6bf02debd7a1cad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:30:57 +0200 Subject: [PATCH 02/85] Read every secret input from a credential file named by NAME_FILE The recipe declared Heimdall's secrets as plain environment, so its admit contract made the operator binding the only place their values could live, and the binding was printed into a transcript. The recipe now declares each secret only as its _FILE twin (GC_ACCESS_DATABASE_URL_FILE required, the rest optional). A binding binds those under [workload.secret_files] to root:root 0400 credential files, and Idunn's rule that a binding binds only declared names leaves a plaintext secret nowhere to go. GC_ACCESS_APP_SHARED_SECRET is deleted outright: nothing reads it. One reader, readSecretInput, serves every secret: it reads NAME_FILE and strips one trailing newline, refuses NAME and NAME_FILE set together, and its errors name the variable and path, never a value or file contents. Storage-backend selection keys off the resolved database URL. The published provider witness and the provider_not_configured body name the _FILE variables, and custody's errors name GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE. The plaintext fallback (NAME read while NAME_FILE is absent) is a named shim for the legacy EnvironmentFile unit only; secret-files Cut 5 deletes it. Idunn secret-files cut, Cut 1. Co-Authored-By: Claude Opus 5.5 --- deployment/idunn/recipe.toml | 23 +++-- src/config.ts | 58 ++++++++++-- src/custody.ts | 6 +- src/providers.ts | 12 +-- tests/config-secrets.test.ts | 174 +++++++++++++++++++++++++++++++++++ 5 files changed, 245 insertions(+), 28 deletions(-) create mode 100644 tests/config-secrets.test.ts diff --git a/deployment/idunn/recipe.toml b/deployment/idunn/recipe.toml index aa1e138..01722d1 100644 --- a/deployment/idunn/recipe.toml +++ b/deployment/idunn/recipe.toml @@ -131,17 +131,20 @@ arguments = [ ] transport = "http" route_required = true +# A secret input is declared only as its `*_FILE` name. The binding binds it +# under [workload.secret_files] to a root:root 0400 credential file, so the value +# never sits in the binding, the unit or a plan. Idunn refuses a binding that +# sets an undeclared name, so a plaintext secret has nowhere to go. optional_environment = [ "CULTMESH_URI_ODIN_RUDP", "GC_ACCESS_APP_BIFROST_BACKEND_CALLBACK_URLS", - "GC_ACCESS_APP_BIFROST_SHARED_SECRET", + "GC_ACCESS_APP_BIFROST_SHARED_SECRET_FILE", "GC_ACCESS_APP_GHOSTLIGHT_BACKEND_CALLBACK_URLS", "GC_ACCESS_APP_GHOSTLIGHT_RUNTIME_IDS", - "GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET", + "GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET_FILE", "GC_ACCESS_APPLY_SCHEMA_ON_STARTUP", - "GC_ACCESS_APP_SHARED_SECRET", "GC_ACCESS_BIFROST_PATRON_SUPPORT_ENDPOINT", - "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET", + "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE", "GC_ACCESS_COMPLETION_TTL_SECONDS", "GC_ACCESS_IDUNN_HEALTH_CONTRACT", "GC_ACCESS_IDUNN_RUDP_HEALTH", @@ -150,27 +153,27 @@ optional_environment = [ "GC_ACCESS_PRIVATE_COMMAND_HOST", "GC_ACCESS_PRIVATE_COMMAND_PORT", "GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID", - "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET", + "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_PATREON_CLIENT_ID", - "GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET", + "GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID", - "GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET", + "GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_ID", - "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET", + "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET_FILE", "GC_ACCESS_SESSION_TTL_SECONDS", "GC_ACCESS_SIGNING_KEY_ID", "GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP", "GC_ACCESS_SIGNING_PRIVATE_KEY_PATH", "GC_ACCESS_STATE_TTL_SECONDS", "GC_ACCESS_STORAGE_BACKEND", - "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64", + "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE", ] required_environment = [ "GAMECULT_IDUNN_RUNTIME_BUNDLE", "GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_PROCESS_WRITE_LEASE", "GC_ACCESS_BASE_URL", - "GC_ACCESS_DATABASE_URL", + "GC_ACCESS_DATABASE_URL_FILE", ] [service.health] diff --git a/src/config.ts b/src/config.ts index 58e5269..285c6b1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { appSlugs, providers, type AppSlug, type Provider } from "./contracts.js"; @@ -120,11 +121,47 @@ function readIdunnCandidateBind( return { host, port }; } +/** + * Read one secret input. Its value lives in a file named by `NAME_FILE`: under + * Idunn that is a systemd credential, so the value never enters a binding, a + * unit, a plan or the process environment. One trailing newline is stripped. + * + * LEGACY SHIM, deleted with the legacy EnvironmentFile unit (secret-files + * Cut 5): when `NAME_FILE` is absent, the plaintext `NAME` is read instead. + * Setting both is refused, so one input has one owner. Under Idunn the recipe + * declares only the `_FILE` names, so the plaintext form cannot be bound there. + * + * Errors name the variable and the path, never a value or a file's contents. + */ +export function readSecretInput(env: NodeJS.ProcessEnv, name: string): string | undefined { + const fileVariable = `${name}_FILE`; + const filePath = env[fileVariable]; + const plaintext = env[name]; + + if (filePath !== undefined && plaintext !== undefined) { + throw new Error(`${name} and ${fileVariable} are both set; set only ${fileVariable}.`); + } + + if (filePath === undefined) { + return plaintext; + } + + let contents: string; + try { + contents = readFileSync(filePath, "utf8"); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code ?? "read error"; + throw new Error(`${fileVariable} names ${filePath}, which could not be read (${code}).`); + } + + return contents.replace(/\r?\n$/, ""); +} + function readProviderConfig(env: NodeJS.ProcessEnv, provider: Provider): ProviderClientConfig { const prefix = `GC_ACCESS_PROVIDER_${provider.toUpperCase()}`; const config: ProviderClientConfig = {}; const clientId = env[`${prefix}_CLIENT_ID`]; - const clientSecret = env[`${prefix}_CLIENT_SECRET`]; + const clientSecret = readSecretInput(env, `${prefix}_CLIENT_SECRET`); if (clientId) { config.clientId = clientId; @@ -200,8 +237,9 @@ export function loadConfig( const issuer = trimTrailingSlash(env.GC_ACCESS_ISSUER ?? publicBaseUrl); const dataRoot = stateRootArgument ?? env.GC_ACCESS_DATA_ROOT ?? path.join(workspaceRoot, ".heimdall-data"); + const databaseUrl = readSecretInput(env, "GC_ACCESS_DATABASE_URL"); const storageBackend = - env.GC_ACCESS_STORAGE_BACKEND === "postgres" || env.GC_ACCESS_DATABASE_URL ? "postgres" : "memory"; + env.GC_ACCESS_STORAGE_BACKEND === "postgres" || databaseUrl ? "postgres" : "memory"; const providersConfig = Object.fromEntries( providers.map((provider) => [provider, readProviderConfig(env, provider)]) ) as Record; @@ -209,7 +247,7 @@ export function loadConfig( appSlugs .map((appSlug) => { const envKey = `GC_ACCESS_APP_${appSlug.toUpperCase()}_SHARED_SECRET`; - return [appSlug, env[envKey]]; + return [appSlug, readSecretInput(env, envKey)]; }) .filter(([, value]) => Boolean(value)) ) as Partial>; @@ -285,20 +323,22 @@ export function loadConfig( config.signingKeyId = env.GC_ACCESS_SIGNING_KEY_ID; } - if (env.GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64) { - config.tokenEncryptionKeyBase64 = env.GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64; + const tokenEncryptionKeyBase64 = readSecretInput(env, "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64"); + if (tokenEncryptionKeyBase64) { + config.tokenEncryptionKeyBase64 = tokenEncryptionKeyBase64; } if (env.GC_ACCESS_BIFROST_PATRON_SUPPORT_ENDPOINT) { config.bifrostPatronSupportEndpoint = env.GC_ACCESS_BIFROST_PATRON_SUPPORT_ENDPOINT; } - if (env.GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET) { - config.bifrostPatronSupportSecret = env.GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET; + const bifrostPatronSupportSecret = readSecretInput(env, "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET"); + if (bifrostPatronSupportSecret) { + config.bifrostPatronSupportSecret = bifrostPatronSupportSecret; } - if (env.GC_ACCESS_DATABASE_URL) { - config.storage.databaseUrl = env.GC_ACCESS_DATABASE_URL; + if (databaseUrl) { + config.storage.databaseUrl = databaseUrl; } return config; diff --git a/src/custody.ts b/src/custody.ts index d2849c1..a70cbe1 100644 --- a/src/custody.ts +++ b/src/custody.ts @@ -14,13 +14,13 @@ export interface TokenCustody { function decodeConfiguredKey(base64Value: string): Buffer { const normalized = base64Value.trim(); if (!normalized) { - throw new Error("GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64 is set but empty."); + throw new Error("GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE is set but empty."); } const key = Buffer.from(normalized, "base64"); if (key.byteLength !== TOKEN_KEY_BYTES) { throw new Error( - `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64 must decode to exactly ${TOKEN_KEY_BYTES} bytes for AES-256-GCM.` + `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE must decode to exactly ${TOKEN_KEY_BYTES} bytes for AES-256-GCM.` ); } @@ -51,7 +51,7 @@ export function createTokenCustody(config: HeimdallConfig): TokenCustody { : (() => { if (config.storage.backend === "postgres") { throw new Error( - "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64 is required when Heimdall uses persistent postgres storage." + "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE is required when Heimdall uses persistent postgres storage." ); } diff --git a/src/providers.ts b/src/providers.ts index e5c3775..506e084 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -21,7 +21,7 @@ export const providerCatalog: Record = { defaultScopes: ["identify", "guilds.members.read"], roles: ["identity", "entitlement"], clientIdEnv: "GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID", - clientSecretEnv: "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET", + clientSecretEnv: "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE", }, patreon: { key: "patreon", @@ -30,7 +30,7 @@ export const providerCatalog: Record = { defaultScopes: ["identity", "identity[email]"], roles: ["identity", "entitlement"], clientIdEnv: "GC_ACCESS_PROVIDER_PATREON_CLIENT_ID", - clientSecretEnv: "GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET", + clientSecretEnv: "GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET_FILE", }, github: { key: "github", @@ -39,7 +39,7 @@ export const providerCatalog: Record = { defaultScopes: ["read:user", "user:email"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_GITHUB_CLIENT_ID", - clientSecretEnv: "GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET", + clientSecretEnv: "GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET_FILE", }, twitch: { key: "twitch", @@ -48,7 +48,7 @@ export const providerCatalog: Record = { defaultScopes: ["user:read:email"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID", - clientSecretEnv: "GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET", + clientSecretEnv: "GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET_FILE", }, youtube: { key: "youtube", @@ -62,7 +62,7 @@ export const providerCatalog: Record = { ], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_ID", - clientSecretEnv: "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET", + clientSecretEnv: "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET_FILE", additionalAuthorizationParams: { access_type: "offline", include_granted_scopes: "true", @@ -80,7 +80,7 @@ export const providerCatalog: Record = { ], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID", - clientSecretEnv: "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET", + clientSecretEnv: "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE", }, }; diff --git a/tests/config-secrets.test.ts b/tests/config-secrets.test.ts new file mode 100644 index 0000000..fa2a56c --- /dev/null +++ b/tests/config-secrets.test.ts @@ -0,0 +1,174 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadConfig, readSecretInput } from "../src/config.js"; +import { appSlugs, providers } from "../src/contracts.js"; +import { providerCatalog, providerExpectedEnv } from "../src/providers.js"; + +// Secret inputs arrive as files named by `NAME_FILE` (systemd credentials under +// Idunn). These tests pin the reader's rules and that every secret Heimdall +// consumes goes through it. + +const tempDirs: string[] = []; + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "heimdall-secret-input-")); + tempDirs.push(dir); + return dir; +} + +function secretFile(contents: string, name = "secret"): string { + const path = join(tempDir(), name); + writeFileSync(path, contents); + return path; +} + +function thrownMessage(action: () => unknown): string { + try { + action(); + } catch (error) { + return (error as Error).message; + } + throw new Error("expected the action to throw"); +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("readSecretInput", () => { + it("reads the value from the file named by NAME_FILE and strips exactly one trailing newline", () => { + const cases: Array<[string, string]> = [ + ["from-the-file\n", "from-the-file"], + ["from-the-file\r\n", "from-the-file"], + ["two-newlines\n\n", "two-newlines\n"], + [" inner spaces kept \n", " inner spaces kept "], + ["no-newline", "no-newline"], + ]; + + for (const [contents, expected] of cases) { + const env = { EXAMPLE_SECRET_FILE: secretFile(contents) }; + expect(readSecretInput(env, "EXAMPLE_SECRET")).toBe(expected); + } + }); + + it("returns undefined when neither form is set", () => { + expect(readSecretInput({}, "EXAMPLE_SECRET")).toBeUndefined(); + }); + + it("falls back to the plaintext name only while NAME_FILE is absent (legacy shim)", () => { + expect(readSecretInput({ EXAMPLE_SECRET: "legacy-plaintext" }, "EXAMPLE_SECRET")).toBe("legacy-plaintext"); + }); + + it("refuses both forms at once, and the refusal carries neither value", () => { + const fileCanary = "canary-file-7c1e0f"; + const plainCanary = "canary-plain-93ab2d"; + const env = { + EXAMPLE_SECRET: plainCanary, + EXAMPLE_SECRET_FILE: secretFile(fileCanary), + }; + + const message = thrownMessage(() => readSecretInput(env, "EXAMPLE_SECRET")); + expect(message).toContain("EXAMPLE_SECRET_FILE"); + expect(message).toMatch(/both set/); + expect(message).not.toContain(fileCanary); + expect(message).not.toContain(plainCanary); + expect(message).not.toContain("canary"); + }); + + it("refuses both forms even when the plaintext form is empty", () => { + const env = { EXAMPLE_SECRET: "", EXAMPLE_SECRET_FILE: secretFile("value") }; + expect(() => readSecretInput(env, "EXAMPLE_SECRET")).toThrow(/both set/); + }); + + it("names the variable and the path of a missing file", () => { + const missing = join(tempDir(), "absent-credential"); + const message = thrownMessage(() => readSecretInput({ EXAMPLE_SECRET_FILE: missing }, "EXAMPLE_SECRET")); + expect(message).toContain("EXAMPLE_SECRET_FILE"); + expect(message).toContain(missing); + expect(message).toContain("ENOENT"); + }); + + it("names the variable and the path of an unreadable file, and carries no byte of what sits beside it", () => { + const dir = tempDir(); + const canary = "canary-sibling-5d2f88"; + writeFileSync(join(dir, "sibling"), canary); + const notAFile = join(dir, "credential"); + mkdirSync(notAFile); + writeFileSync(join(notAFile, "inside"), canary); + + const message = thrownMessage(() => readSecretInput({ EXAMPLE_SECRET_FILE: notAFile }, "EXAMPLE_SECRET")); + expect(message).toContain("EXAMPLE_SECRET_FILE"); + expect(message).toContain(notAFile); + expect(message).not.toContain("canary"); + }); +}); + +describe("loadConfig secret inputs", () => { + it("configures every secret Heimdall consumes from its _FILE twin alone", () => { + const env: Record = {}; + const expected = new Map(); + function bind(name: string): string { + const value = `value-of-${name.toLowerCase()}`; + env[`${name}_FILE`] = secretFile(`${value}\n`, name); + expected.set(name, value); + return value; + } + + const databaseUrl = bind("GC_ACCESS_DATABASE_URL"); + const tokenKey = bind("GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64"); + const patronSecret = bind("GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET"); + const providerSecrets = Object.fromEntries( + providers.map((provider) => [provider, bind(`GC_ACCESS_PROVIDER_${provider.toUpperCase()}_CLIENT_SECRET`)]) + ); + const appSecrets = Object.fromEntries( + appSlugs.map((slug) => [slug, bind(`GC_ACCESS_APP_${slug.toUpperCase()}_SHARED_SECRET`)]) + ); + + const config = loadConfig(env, []); + + expect(config.storage.databaseUrl).toBe(databaseUrl); + expect(config.tokenEncryptionKeyBase64).toBe(tokenKey); + expect(config.bifrostPatronSupportSecret).toBe(patronSecret); + for (const provider of providers) { + expect(config.providers[provider].clientSecret).toBe(providerSecrets[provider]); + } + expect(config.appSharedSecrets).toEqual(appSecrets); + }); + + it("selects postgres storage from a database URL given only as a file", () => { + const config = loadConfig( + { GC_ACCESS_DATABASE_URL_FILE: secretFile("postgres://heimdall@localhost/heimdall\n") }, + [] + ); + expect(config.storage.backend).toBe("postgres"); + }); + + it("refuses to start when the database URL file is absent", () => { + const missing = join(tempDir(), "database-url"); + const message = thrownMessage(() => loadConfig({ GC_ACCESS_DATABASE_URL_FILE: missing }, [])); + expect(message).toContain("GC_ACCESS_DATABASE_URL_FILE"); + expect(message).toContain(missing); + }); + + it("refuses a secret bound both ways at startup", () => { + const env = { + GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64: "plain", + GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE: secretFile("file"), + }; + expect(() => loadConfig(env, [])).toThrow(/GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE/); + }); + + it("publishes provider secret variable names that actually configure the secret", () => { + for (const provider of providers) { + const secretVariable = providerCatalog[provider].clientSecretEnv; + expect(providerExpectedEnv(provider)).toContain(secretVariable); + + const config = loadConfig({ [secretVariable]: secretFile(`${provider}-secret\n`) }, []); + expect(config.providers[provider].clientSecret).toBe(`${provider}-secret`); + } + }); +}); From 9f0cd73299e26ca8a5fae32f0a6f6e4c24bba4d5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:49:23 +0200 Subject: [PATCH 03/85] Pin the newline and storage-selection rules StrykerJS found undefended StrykerJS on the reader and its call sites left two findings about the tests. Every trim fixture had its newline only at the end, so stripping the first newline anywhere survived: a two-line value now pins "one trailing newline". No fixture reached the memory branch of storage selection, so forcing postgres, dropping the backend check or inverting it survived: the memory default and an explicit backend of either kind are now pinned. Idunn secret-files cut, Cut 1. Co-Authored-By: Claude Opus 5.5 --- tests/config-secrets.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/config-secrets.test.ts b/tests/config-secrets.test.ts index fa2a56c..e58e6cf 100644 --- a/tests/config-secrets.test.ts +++ b/tests/config-secrets.test.ts @@ -45,6 +45,7 @@ describe("readSecretInput", () => { ["from-the-file\n", "from-the-file"], ["from-the-file\r\n", "from-the-file"], ["two-newlines\n\n", "two-newlines\n"], + ["line-one\nline-two\n", "line-one\nline-two"], [" inner spaces kept \n", " inner spaces kept "], ["no-newline", "no-newline"], ]; @@ -147,6 +148,12 @@ describe("loadConfig secret inputs", () => { expect(config.storage.backend).toBe("postgres"); }); + it("keeps memory storage when no database URL is bound, and postgres when it is asked for", () => { + expect(loadConfig({}, []).storage.backend).toBe("memory"); + expect(loadConfig({ GC_ACCESS_STORAGE_BACKEND: "memory" }, []).storage.backend).toBe("memory"); + expect(loadConfig({ GC_ACCESS_STORAGE_BACKEND: "postgres" }, []).storage.backend).toBe("postgres"); + }); + it("refuses to start when the database URL file is absent", () => { const missing = join(tempDir(), "database-url"); const message = thrownMessage(() => loadConfig({ GC_ACCESS_DATABASE_URL_FILE: missing }, [])); From 0178d071c1f8d2262ba91789b9476d53f5b01d84 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 13:50:53 +0200 Subject: [PATCH 04/85] readSecretInput: never print the path, refuse relative paths and blank secrets Soul F1: the unreadable-file error carried the NAME_FILE value, and Idunn admits any string for a declared name, so a URL bound into GC_ACCESS_DATABASE_URL_FILE by mistake was printed in full at startup. Errors now name only the variable and an error code, and a NAME_FILE that is not an absolute path is refused naming the variable alone. Soul F2 and F6: a blank database URL file selected memory storage silently, and a whitespace one selected postgres with a URL of " ". The reader now refuses an empty or whitespace-only value from either form, so custody's own "set but empty" check is redundant and is deleted; a short key still fails its length check. The legacy service.env holds no blank secret-class value (checked as name/blank booleans), so the legacy unit is unaffected. Soul F3: tests pin that an empty NAME_FILE is refused rather than unset (S1), and that each call site (token key, provider secret, app shared secret, database URL, patron secret) refuses both forms, so no site can read the plaintext name ahead of the reader (S6, S7, S8). Co-Authored-By: Claude Opus 5.5 --- src/config.ts | 37 +++++++++++++----- src/custody.ts | 8 +--- tests/config-secrets.test.ts | 76 ++++++++++++++++++++++++++++-------- 3 files changed, 89 insertions(+), 32 deletions(-) diff --git a/src/config.ts b/src/config.ts index 285c6b1..c31e0e8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -131,7 +131,12 @@ function readIdunnCandidateBind( * Setting both is refused, so one input has one owner. Under Idunn the recipe * declares only the `_FILE` names, so the plaintext form cannot be bound there. * - * Errors name the variable and the path, never a value or a file's contents. + * `NAME_FILE` must be an absolute path, and a value that is empty or only + * whitespace is refused: a blank secret is a misconfiguration, never "unset". + * + * Errors name only the variable and an error code. They never carry the path, + * a value or a file's contents: Idunn admits any string for a declared name, so + * a secret bound into a `_FILE` slot by mistake would otherwise be printed. */ export function readSecretInput(env: NodeJS.ProcessEnv, name: string): string | undefined { const fileVariable = `${name}_FILE`; @@ -142,19 +147,31 @@ export function readSecretInput(env: NodeJS.ProcessEnv, name: string): string | throw new Error(`${name} and ${fileVariable} are both set; set only ${fileVariable}.`); } + let value: string; + let source: string; if (filePath === undefined) { - return plaintext; + if (plaintext === undefined) { + return undefined; + } + value = plaintext; + source = name; + } else { + if (!path.isAbsolute(filePath)) { + throw new Error(`${fileVariable} is not an absolute path.`); + } + try { + value = readFileSync(filePath, "utf8").replace(/\r?\n$/, ""); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code ?? "read error"; + throw new Error(`${fileVariable} could not be read (${code}).`); + } + source = fileVariable; } - let contents: string; - try { - contents = readFileSync(filePath, "utf8"); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code ?? "read error"; - throw new Error(`${fileVariable} names ${filePath}, which could not be read (${code}).`); + if (value.trim() === "") { + throw new Error(`${source} is empty.`); } - - return contents.replace(/\r?\n$/, ""); + return value; } function readProviderConfig(env: NodeJS.ProcessEnv, provider: Provider): ProviderClientConfig { diff --git a/src/custody.ts b/src/custody.ts index a70cbe1..baa45b0 100644 --- a/src/custody.ts +++ b/src/custody.ts @@ -12,12 +12,8 @@ export interface TokenCustody { } function decodeConfiguredKey(base64Value: string): Buffer { - const normalized = base64Value.trim(); - if (!normalized) { - throw new Error("GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE is set but empty."); - } - - const key = Buffer.from(normalized, "base64"); + // readSecretInput refuses a blank key; a short one fails the length check. + const key = Buffer.from(base64Value.trim(), "base64"); if (key.byteLength !== TOKEN_KEY_BYTES) { throw new Error( `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE must decode to exactly ${TOKEN_KEY_BYTES} bytes for AES-256-GCM.` diff --git a/tests/config-secrets.test.ts b/tests/config-secrets.test.ts index e58e6cf..9cbe902 100644 --- a/tests/config-secrets.test.ts +++ b/tests/config-secrets.test.ts @@ -85,15 +85,27 @@ describe("readSecretInput", () => { expect(() => readSecretInput(env, "EXAMPLE_SECRET")).toThrow(/both set/); }); - it("names the variable and the path of a missing file", () => { - const missing = join(tempDir(), "absent-credential"); - const message = thrownMessage(() => readSecretInput({ EXAMPLE_SECRET_FILE: missing }, "EXAMPLE_SECRET")); + it("refuses an unreadable path naming only the variable and the code, not the path", () => { + // A secret bound into a _FILE slot by mistake must not be printed back. + const canaryPath = join(tempDir(), "canary-path-3b7e1d"); + const message = thrownMessage(() => readSecretInput({ EXAMPLE_SECRET_FILE: canaryPath }, "EXAMPLE_SECRET")); expect(message).toContain("EXAMPLE_SECRET_FILE"); - expect(message).toContain(missing); expect(message).toContain("ENOENT"); + expect(message).not.toContain("canary"); }); - it("names the variable and the path of an unreadable file, and carries no byte of what sits beside it", () => { + it("refuses a _FILE value that is not an absolute path, without echoing it", () => { + for (const value of ["postgres://heimdall:canary-pw-6f02aa@db/heimdall", "relative/canary-rel-0c9d", ""]) { + const message = thrownMessage(() => readSecretInput({ EXAMPLE_SECRET_FILE: value }, "EXAMPLE_SECRET")); + expect(message).toBe("EXAMPLE_SECRET_FILE is not an absolute path."); + } + }); + + it("does not treat an empty _FILE as unset, even with no plaintext form", () => { + expect(() => readSecretInput({ EXAMPLE_SECRET_FILE: "" }, "EXAMPLE_SECRET")).toThrow(/EXAMPLE_SECRET_FILE/); + }); + + it("refuses an unreadable file, and carries no byte of what sits in or beside it", () => { const dir = tempDir(); const canary = "canary-sibling-5d2f88"; writeFileSync(join(dir, "sibling"), canary); @@ -102,9 +114,27 @@ describe("readSecretInput", () => { writeFileSync(join(notAFile, "inside"), canary); const message = thrownMessage(() => readSecretInput({ EXAMPLE_SECRET_FILE: notAFile }, "EXAMPLE_SECRET")); - expect(message).toContain("EXAMPLE_SECRET_FILE"); - expect(message).toContain(notAFile); - expect(message).not.toContain("canary"); + expect(message).toBe("EXAMPLE_SECRET_FILE could not be read (EISDIR)."); + }); + + it("refuses a blank secret file rather than treating it as unset", () => { + for (const contents of ["", "\n", " \n", "\t\r\n", " "]) { + const env = { EXAMPLE_SECRET_FILE: secretFile(contents) }; + expect(thrownMessage(() => readSecretInput(env, "EXAMPLE_SECRET"))).toBe("EXAMPLE_SECRET_FILE is empty."); + } + }); + + it("refuses a blank plaintext secret too", () => { + for (const value of ["", " ", "\n"]) { + expect(thrownMessage(() => readSecretInput({ EXAMPLE_SECRET: value }, "EXAMPLE_SECRET"))).toBe( + "EXAMPLE_SECRET is empty." + ); + } + }); + + it("keeps a value whose only whitespace is inside or around real content", () => { + const env = { EXAMPLE_SECRET_FILE: secretFile(" padded value \n") }; + expect(readSecretInput(env, "EXAMPLE_SECRET")).toBe(" padded value "); }); }); @@ -157,18 +187,32 @@ describe("loadConfig secret inputs", () => { it("refuses to start when the database URL file is absent", () => { const missing = join(tempDir(), "database-url"); const message = thrownMessage(() => loadConfig({ GC_ACCESS_DATABASE_URL_FILE: missing }, [])); - expect(message).toContain("GC_ACCESS_DATABASE_URL_FILE"); - expect(message).toContain(missing); + expect(message).toBe("GC_ACCESS_DATABASE_URL_FILE could not be read (ENOENT)."); }); - it("refuses a secret bound both ways at startup", () => { - const env = { - GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64: "plain", - GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE: secretFile("file"), - }; - expect(() => loadConfig(env, [])).toThrow(/GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE/); + it("refuses to start on a blank database URL file instead of falling back to memory storage", () => { + for (const contents of ["", "\n", " \n"]) { + const env = { GC_ACCESS_DATABASE_URL_FILE: secretFile(contents) }; + expect(() => loadConfig(env, [])).toThrow("GC_ACCESS_DATABASE_URL_FILE is empty."); + } }); + // One per call site: a site that reads the plaintext name itself before the + // reader would let the plaintext win silently instead of refusing. + const bothSetSites: Array<[string, string]> = [ + ["token encryption key", "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64"], + ["provider client secret", "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET"], + ["app shared secret", "GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET"], + ["database URL", "GC_ACCESS_DATABASE_URL"], + ["patron support secret", "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET"], + ]; + for (const [site, name] of bothSetSites) { + it(`refuses the ${site} bound both ways at startup`, () => { + const env = { [name]: "plain", [`${name}_FILE`]: secretFile("file") }; + expect(() => loadConfig(env, [])).toThrow(`${name} and ${name}_FILE are both set`); + }); + } + it("publishes provider secret variable names that actually configure the secret", () => { for (const provider of providers) { const secretVariable = providerCatalog[provider].clientSecretEnv; From f8ff2b26ba076a101c8b2ef35ce158e60d8d4ecd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 13:51:18 +0200 Subject: [PATCH 05/85] Declare the GitHub and Spotify client-secret files the witness publishes Soul F5: the provider witness names GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET_FILE and GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE, but the recipe declared neither, so an Idunn binding could not configure either provider. Both providers are live code (the provider list, the catalog and the OAuth start route serve them), so both names are declared as optional. Co-Authored-By: Claude Opus 5.5 --- deployment/idunn/recipe.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/idunn/recipe.toml b/deployment/idunn/recipe.toml index 01722d1..d18a836 100644 --- a/deployment/idunn/recipe.toml +++ b/deployment/idunn/recipe.toml @@ -154,8 +154,10 @@ optional_environment = [ "GC_ACCESS_PRIVATE_COMMAND_PORT", "GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID", "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE", + "GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_PATREON_CLIENT_ID", "GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET_FILE", + "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID", "GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_ID", From fe26208f052da6181c79b24053103835749990e0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 13:52:11 +0200 Subject: [PATCH 06/85] Docs and .env.example teach secrets as NAME_FILE credential files Soul F4: the architecture configuration list, the Spotiverse restore steps and .env.example still taught plaintext secret variables. The architecture list now names only what src/config.ts reads, plain settings apart from secret files, and drops the names nothing reads (GC_ACCESS_SESSION_SECRET, GC_ACCESS_ENABLED, GC_ACCESS_INTERNAL_URL, the Discord bot token, the Patreon campaign id and two TTLs). .env.example comments out the _FILE lines, because an empty NAME_FILE= is refused at startup. Co-Authored-By: Claude Opus 5.5 --- .env.example | 22 ++++++++------- docs/app-profiles/spotiverse.md | 6 ++--- docs/architecture.md | 47 +++++++++++++++++---------------- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/.env.example b/.env.example index 8d91415..999a299 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,5 @@ +# Secrets are files: set NAME_FILE to the absolute path of a file holding only +# the value. An empty NAME_FILE= is refused, so unused ones stay commented out. HOST=127.0.0.1 PORT=4100 GC_ACCESS_PRIVATE_COMMAND_HOST=127.0.0.1 @@ -5,7 +7,7 @@ GC_ACCESS_PRIVATE_COMMAND_PORT=4101 GC_ACCESS_BASE_URL=http://127.0.0.1:4100 GC_ACCESS_ISSUER=http://127.0.0.1:4100 GC_ACCESS_STORAGE_BACKEND=memory -GC_ACCESS_DATABASE_URL= +# GC_ACCESS_DATABASE_URL_FILE=/absolute/path/to/credential GC_ACCESS_APPLY_SCHEMA_ON_STARTUP=1 GC_ACCESS_SESSION_TTL_SECONDS=3600 GC_ACCESS_STATE_TTL_SECONDS=600 @@ -13,22 +15,22 @@ GC_ACCESS_COMPLETION_TTL_SECONDS=300 GC_ACCESS_SIGNING_PRIVATE_KEY_PATH= GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=0 GC_ACCESS_SIGNING_KEY_ID= -GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64= +# GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE=/absolute/path/to/credential GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID= -GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET= +# GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_PROVIDER_PATREON_CLIENT_ID= -GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET= +# GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_PROVIDER_GITHUB_CLIENT_ID= -GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET= +# GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID= -GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET= +# GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_ID= -GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET= +# GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID= -GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET= +# GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE=/absolute/path/to/credential -GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET= +# GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_APP_SPOTIVERSE_BACKEND_CALLBACK_URLS= -GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET= +# GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_APP_GHOSTLIGHT_RUNTIME_IDS=yggdrasil-ghostlight diff --git a/docs/app-profiles/spotiverse.md b/docs/app-profiles/spotiverse.md index ed52874..64a71eb 100644 --- a/docs/app-profiles/spotiverse.md +++ b/docs/app-profiles/spotiverse.md @@ -38,7 +38,7 @@ Everything below this line is still accurate. To bring Spotiverse back: **`http://127.0.0.1:8796/auth/heimdall/callback`**; that exact URL was the allowlisted value. 3. Configure `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID` and - `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET` in the host service environment. + `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE` pointing at a credential file. They were removed from `/srv/heimdall/env/service.env` on retirement, so a fresh Spotify application registration may be needed. 4. Restore the three tests from `5a15f71` if the capabilities they covered @@ -80,8 +80,8 @@ Heimdall needs: ```text GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID=... -GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET=... -GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET=... +GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE=/absolute/path/to/spotify-client-secret +GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET_FILE=/absolute/path/to/spotiverse-shared-secret GC_ACCESS_APP_SPOTIVERSE_BACKEND_CALLBACK_URLS=https://your-spotiverse-portal.example/auth/heimdall/callback ``` diff --git a/docs/architecture.md b/docs/architecture.md index ed1abc4..88bfeaa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -572,34 +572,35 @@ every app's audience or product data. ## Configuration split -Suggested generic env surface: +Heimdall reads plain settings from its environment and every secret from a +file. A secret input `NAME` is configured as `NAME_FILE`, the absolute path of +a file holding only that value; under Idunn the binding binds it in +`[workload.secret_files]` to a `root:root 0400` credential. A blank file is +refused at startup, and so is setting both `NAME` and `NAME_FILE`. + +Plain settings: -- `GC_ACCESS_ENABLED=1` - `GC_ACCESS_BASE_URL=https://heimdall.gamecult.org` -- `GC_ACCESS_INTERNAL_URL=http://127.0.0.1:4100` -- `GC_ACCESS_SESSION_SECRET=...` -- `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH=/var/lib/heimdall/signing-key.pem` +- `GC_ACCESS_ISSUER=https://heimdall.gamecult.org` +- `GC_ACCESS_STORAGE_BACKEND=postgres` +- `GC_ACCESS_SIGNING_PRIVATE_KEY_PATH=/etc/gamecult/heimdall/credentials/signing-key.pem` - `GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1` - `GC_ACCESS_SIGNING_KEY_ID=...` -- `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64=...` - `GC_ACCESS_SESSION_TTL_SECONDS=3600` - `GC_ACCESS_STATE_TTL_SECONDS=600` -- `GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID=...` -- `GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET=...` -- `GC_ACCESS_PROVIDER_DISCORD_BOT_TOKEN=...` -- `GC_ACCESS_PROVIDER_PATREON_CLIENT_ID=...` -- `GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET=...` -- `GC_ACCESS_PROVIDER_PATREON_CAMPAIGN_ID=...` -- `GC_ACCESS_PROVIDER_GITHUB_CLIENT_ID=...` -- `GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET=...` -- `GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID=...` -- `GC_ACCESS_PROVIDER_TWITCH_CLIENT_SECRET=...` -- `GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_ID=...` -- `GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_SECRET=...` -- `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID=...` -- `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET=...` -- `GC_ACCESS_ENTITLEMENT_CACHE_TTL_SECONDS=900` -- `GC_ACCESS_PROVIDER_FAILURE_GRACE_SECONDS=3600` +- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,GITHUB,TWITCH,YOUTUBE,SPOTIFY}_CLIENT_ID=...` +- `GC_ACCESS_APP_{APP_SLUG}_BACKEND_CALLBACK_URLS=...` + +Secret files: + +- `GC_ACCESS_DATABASE_URL_FILE=/etc/gamecult/heimdall/credentials/GC_ACCESS_DATABASE_URL` +- `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE=...` +- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,GITHUB,TWITCH,YOUTUBE,SPOTIFY}_CLIENT_SECRET_FILE=...` +- `GC_ACCESS_APP_{APP_SLUG}_SHARED_SECRET_FILE=...` +- `GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE=...` + +`src/config.ts` is the complete list; `deployment/idunn/recipe.toml` declares +the subset an Idunn binding may set. Only configure the providers Heimdall actually owns for a given deployment. @@ -610,7 +611,7 @@ ephemeral dev signing key. That is acceptable for local iteration and absolutely not acceptable for real deployment. Managed provider tokens should be sealed at rest with a stable -`GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64` secret. The current runtime will still +`GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE` secret. The current runtime will still mint an ephemeral dev token key for in-memory local work, but persistent Postgres deployments should treat a missing token-encryption key as a hard misconfiguration, not a vibe. From e35441e37854f76500f193af275d5a714703fba6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:03:24 +0200 Subject: [PATCH 07/85] Pin the token-key length refusal custody now relies on With custody's "set but empty" check deleted, the 32-byte length check is the only refusal of a malformed token key, and StrykerJS showed nothing tested it (forcing it off survived). A 16-, 31- and 33-byte key are each refused, and the refusal does not carry the key. Co-Authored-By: Claude Opus 5.5 --- tests/security.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/security.test.ts b/tests/security.test.ts index 134054b..5b07c6d 100644 --- a/tests/security.test.ts +++ b/tests/security.test.ts @@ -64,6 +64,22 @@ describe("Heimdall security helpers", () => { expect(custody.decrypt(encrypted)).toBe("discord-access-token"); }); + it("refuses a token key that does not decode to 32 bytes, without echoing it", () => { + for (const length of [16, 31, 33]) { + const key = Buffer.alloc(length, 7).toString("base64"); + const message = (() => { + try { + createTokenCustody({ ...createTestConfig(), tokenEncryptionKeyBase64: key }); + } catch (error) { + return (error as Error).message; + } + throw new Error(`a ${length}-byte key was accepted`); + })(); + expect(message).toMatch(/GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE must decode to exactly 32 bytes/); + expect(message).not.toContain(key); + } + }); + it("refuses persistent provider-token storage without a stable encryption key", () => { expect(() => createTokenCustody( From e091bb3fd1b30fcd1ef536526014c063d7b0a46f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:40:07 +0200 Subject: [PATCH 08/85] Signing key path: absolute only, and no error carries the path Soul S3: GC_ACCESS_SIGNING_PRIVATE_KEY_PATH is now the only way the signing key gets in, and its errors printed the path: the missing-file message interpolated it, and fs errors from the read and the bootstrap mkdir carried it in their message and properties. An operator moving off the deleted inline PEM input who binds the key text to _PATH would have the private key written to the journal. The path now takes the secret reader's rules. loadConfig refuses a relative value, which PEM text always is. Every signing-key failure is a fresh error naming only the variable and a code, with no cause and no properties. Co-Authored-By: Claude Opus 5.5 --- src/config.ts | 12 ++++++++++-- src/signing.ts | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index c31e0e8..39ab96f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -332,8 +332,16 @@ export function loadConfig( config.odinCultMeshUri = odinCultMeshUri; } - if (env.GC_ACCESS_SIGNING_PRIVATE_KEY_PATH) { - config.signingPrivateKeyPath = env.GC_ACCESS_SIGNING_PRIVATE_KEY_PATH; + // The signing key path follows the secret reader's rules: it must be + // absolute, and no error repeats it. An operator moving off the deleted + // inline PEM input who binds the key text here would otherwise have the + // private key printed back into the journal. + const signingPrivateKeyPath = env.GC_ACCESS_SIGNING_PRIVATE_KEY_PATH; + if (signingPrivateKeyPath) { + if (!path.isAbsolute(signingPrivateKeyPath)) { + throw new Error("GC_ACCESS_SIGNING_PRIVATE_KEY_PATH is not an absolute path."); + } + config.signingPrivateKeyPath = signingPrivateKeyPath; } if (env.GC_ACCESS_SIGNING_KEY_ID) { diff --git a/src/signing.ts b/src/signing.ts index 1a41a97..b7d72aa 100644 --- a/src/signing.ts +++ b/src/signing.ts @@ -56,8 +56,30 @@ function exportPrivateKeyPem(privateKey: KeyObject): string { return typeof exported === "string" ? exported : exported.toString("utf8"); } +const SIGNING_KEY_VARIABLE = "GC_ACCESS_SIGNING_PRIVATE_KEY_PATH"; + +/** + * Every signing-key failure names only the variable and an error code. It + * carries no path, no file contents and no cause, because Node prints an + * error's cause and properties along with its message. + */ +function signingKeyError(failure: string, error: unknown): Error { + const code = (error as NodeJS.ErrnoException | undefined)?.code ?? "error"; + return new Error(`${SIGNING_KEY_VARIABLE} ${failure} (${code}).`); +} + function loadPrivateKeyFromPath(path: string): KeyObject { - return createPrivateKey(readFileSync(path, "utf8")); + let pem: string; + try { + pem = readFileSync(path, "utf8"); + } catch (error) { + throw signingKeyError("could not be read", error); + } + try { + return createPrivateKey(pem); + } catch (error) { + throw signingKeyError("does not hold a usable private key", error); + } } function createOrLoadPrivateKeyFromPath(config: HeimdallConfig): { @@ -81,12 +103,16 @@ function createOrLoadPrivateKeyFromPath(config: HeimdallConfig): { if (!config.bootstrapSigningPrivateKeyOnMissing) { throw new Error( - `Signing key file '${path}' does not exist. Set GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1 to create it on first boot.` + `${SIGNING_KEY_VARIABLE} names no file (ENOENT). Set GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP=1 to create it on first boot.` ); } const privateKey = generateKeyPairSync("ed25519").privateKey; - mkdirSync(dirname(path), { recursive: true }); + try { + mkdirSync(dirname(path), { recursive: true }); + } catch (error) { + throw signingKeyError("could not be created", error); + } try { writeFileSync(path, exportPrivateKeyPem(privateKey), { encoding: "utf8", @@ -99,7 +125,7 @@ function createOrLoadPrivateKeyFromPath(config: HeimdallConfig): { }; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") { - throw error; + throw signingKeyError("could not be created", error); } return { From d73a26dfd15f9813775d6e35a02091b4f43c1e26 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:40:21 +0200 Subject: [PATCH 09/85] Pin no-leak at the terminal: run the real entrypoint with canaries Soul S1 and S5: the secret tests checked error.message, but Node prints an error's cause and properties too, and nothing tested src/index.ts. Soul's mutants MA (the read error given `{ cause: error }`) and MI (the startup catch printing the config) both leaked a canary to stderr with the suite green. tests/entrypoint-secrets.test.ts spawns `tsx src/index.ts` with only the given environment and fails on any byte of the canary on stdout or stderr. Each case must also stop by itself, fail, and say its expected refusal, so none can pass by never reaching its path. Cases: a URL in an absolute _FILE slot, a _FILE under a regular file, a relative _FILE, both forms set, a blank file, a short token key, four malformed or refused database URLs in a real file (pinning today's pg, which drops Node's `input` property), five signing-key failures including the bootstrap mkdir, and a listen failure with every secret configured, which is the entrypoint catch that holds the config. Co-Authored-By: Claude Opus 5.5 --- tests/entrypoint-secrets.test.ts | 194 +++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 tests/entrypoint-secrets.test.ts diff --git a/tests/entrypoint-secrets.test.ts b/tests/entrypoint-secrets.test.ts new file mode 100644 index 0000000..1be4442 --- /dev/null +++ b/tests/entrypoint-secrets.test.ts @@ -0,0 +1,194 @@ +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, describe, expect, it } from "vitest"; + +// A secret must never reach a terminal. Under Idunn, Heimdall's stderr goes to +// the journal and the install script tails the journal to the operator's +// terminal, so the layer that matters is the process's own output, not an +// error's message. These cases run the real entrypoint with canary values in +// the places a misbinding puts secrets, and fail on any byte of the canary +// anywhere on stdout or stderr: in a message, an error cause, an error +// property, or anything else Node chooses to print. + +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); +const tsxCli = createRequire(import.meta.url).resolve("tsx/cli"); +const scratch = mkdtempSync(join(tmpdir(), "heimdall-entrypoint-")); +const CANARY = "CANARY"; + +afterAll(() => { + rmSync(scratch, { recursive: true, force: true }); +}); + +function file(name: string, contents: string): string { + const path = join(scratch, name); + writeFileSync(path, contents); + return path; +} + +interface Run { + code: number | null; + signal: NodeJS.Signals | null; + output: string; +} + +/** Runs `tsx src/index.ts` with only the given environment, merging stdout and stderr. */ +function runEntrypoint(name: string, env: Record): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [tsxCli, "src/index.ts"], { + cwd: repoRoot, + env: { + PATH: process.env.PATH ?? "", + HOME: scratch, + GC_ACCESS_DATA_ROOT: join(scratch, `data-${name}`), + PORT: "0", + GC_ACCESS_PRIVATE_COMMAND_PORT: "0", + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (chunk: Buffer) => (output += chunk.toString("utf8"))); + child.stderr.on("data", (chunk: Buffer) => (output += chunk.toString("utf8"))); + // A case that fails to fail would start serving; stop it rather than hang. + const timer = setTimeout(() => child.kill("SIGKILL"), 30_000); + child.on("error", reject); + child.on("close", (code, signal) => { + clearTimeout(timer); + resolve({ code, signal, output }); + }); + }); +} + +/** A base64 value that decodes to exactly 32 bytes and still carries the canary. */ +function canaryTokenKey(tag: string): string { + return `${CANARY}${tag}`.padEnd(43, "A") + "="; +} + +interface Case { + name: string; + env: () => Record; + /** Something the failure must say, so a case cannot pass by never reaching its path. */ + expect: RegExp; +} + +const cases: Case[] = [ + { + name: "a URL bound into an absolute _FILE slot", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: `/postgres://heimdall:${CANARY}url@db/heimdall` }), + expect: /GC_ACCESS_DATABASE_URL_FILE could not be read \(ENOENT\)/, + }, + { + name: "a _FILE path under a regular file", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: join(file("plain", "x"), `${CANARY}notdir`) }), + expect: /GC_ACCESS_DATABASE_URL_FILE could not be read \(ENOTDIR\)/, + }, + { + name: "a secret bound into a relative _FILE slot", + env: () => ({ GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE: `${CANARY}relative-secret` }), + expect: /GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE is not an absolute path/, + }, + { + name: "a secret bound both ways", + env: () => ({ + GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET: `${CANARY}plain`, + GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET_FILE: file("both", `${CANARY}file\n`), + }), + expect: /GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET and GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET_FILE are both set/, + }, + { + name: "a blank secret file", + env: () => ({ GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE: join(scratch, "blank") }), + expect: /GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE is empty/, + }, + { + name: "a token key that decodes short", + env: () => ({ GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE: file("short-key", `${CANARY}shortkey\n`) }), + expect: /GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE must decode to exactly 32 bytes/, + }, + { + name: "a database URL with an unusable port", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-port", `postgres://heimdall:${CANARY}port@db:notaport/heimdall\n`) }), + expect: /./, + }, + { + name: "a database URL that does not parse", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-parse", `postgres://heimdall:${CANARY} parse@[bad/heimdall\n`) }), + expect: /./, + }, + { + name: "a database URL with a broken percent escape", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-percent", `postgres://heimdall:${CANARY}%zz@127.0.0.1:1/heimdall\n`) }), + expect: /./, + }, + { + name: "a database that refuses the connection", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-refused", `postgres://heimdall:${CANARY}refused@127.0.0.1:1/heimdall\n`) }), + expect: /ECONNREFUSED/, + }, + { + name: "private key text bound to the signing key path", + env: () => ({ + GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: `-----BEGIN PRIVATE KEY-----\n${CANARY}pem\n-----END PRIVATE KEY-----\n`, + }), + expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH is not an absolute path/, + }, + { + name: "a signing key path that names no file", + env: () => ({ GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: join(scratch, `${CANARY}missing-key`) }), + expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH names no file \(ENOENT\)/, + }, + { + name: "a signing key file that is not a key", + env: () => ({ GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: file("not-a-key", `${CANARY}notakey\n`) }), + expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH does not hold a usable private key/, + }, + { + name: "a signing key path that is a directory", + env: () => { + const dir = join(scratch, "key-dir"); + mkdirSync(dir, { recursive: true }); + return { GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: dir }; + }, + expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH could not be read \(EISDIR\)/, + }, + { + name: "a bootstrapped signing key whose directory cannot be created", + env: () => ({ + GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: join(file("key-parent", "x"), `${CANARY}dir`, "key.pem"), + GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP: "1", + }), + expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH could not be created \(ENOTDIR\)/, + }, + { + // Every secret is configured and valid, and startup fails afterwards, in + // the entrypoint's own catch: the one place that holds the whole config. + name: "a listen failure with every secret configured", + env: () => ({ + HOST: "203.0.113.1", + GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE: file("listen-key", `${canaryTokenKey("key")}\n`), + GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE: file("listen-patron", `${CANARY}patron\n`), + GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET_FILE: file("listen-github", `${CANARY}github\n`), + GC_ACCESS_APP_BIFROST_SHARED_SECRET_FILE: file("listen-bifrost", `${CANARY}bifrost\n`), + }), + expect: /Heimdall failed to start/, + }, +]; + +describe("the entrypoint never prints a secret", () => { + writeFileSync(join(scratch, "blank"), " \n"); + + for (const testCase of cases) { + it(`refuses ${testCase.name} without printing the canary`, async () => { + const run = await runEntrypoint(testCase.name.replace(/\W+/g, "-"), testCase.env()); + + expect(run.signal, "the process must stop by itself").toBeNull(); + expect(run.code, "startup must fail").not.toBe(0); + expect(run.output).toMatch(testCase.expect); + expect(run.output.includes(CANARY), "no byte of the canary may reach the terminal").toBe(false); + }, 45_000); + } +}); From 2800d88d105cb9de225e948bca12d8374998c9dc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:40:21 +0200 Subject: [PATCH 10/85] Pin the recipe's secret declarations, and declare every app secret file Soul S2: no test read deployment/idunn/recipe.toml, so a plaintext secret name could be declared again (mutant ME) or a published _FILE name dropped (mutant MF) with the suite green. tests/recipe-secrets.test.ts reads the service's declared names and asserts that no secret-shaped name is declared except as _FILE, that none of the secrets Heimdall reads is declared under its plaintext name, and that every provider client-secret file the witness publishes, every published app profile's shared-secret file, and the database URL, token key and patron secret files are declared. The app-profile rule found two gaps: the witness publishes the repixelizer and streampixels profiles and Heimdall reads their shared-secret files, but the recipe declared neither, so an Idunn binding could not configure them. Both are declared as optional, as GitHub and Spotify were for Soul F5. Co-Authored-By: Claude Opus 5.5 --- deployment/idunn/recipe.toml | 2 + tests/recipe-secrets.test.ts | 78 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 tests/recipe-secrets.test.ts diff --git a/deployment/idunn/recipe.toml b/deployment/idunn/recipe.toml index d18a836..8524451 100644 --- a/deployment/idunn/recipe.toml +++ b/deployment/idunn/recipe.toml @@ -142,6 +142,8 @@ optional_environment = [ "GC_ACCESS_APP_GHOSTLIGHT_BACKEND_CALLBACK_URLS", "GC_ACCESS_APP_GHOSTLIGHT_RUNTIME_IDS", "GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET_FILE", + "GC_ACCESS_APP_REPIXELIZER_SHARED_SECRET_FILE", + "GC_ACCESS_APP_STREAMPIXELS_SHARED_SECRET_FILE", "GC_ACCESS_APPLY_SCHEMA_ON_STARTUP", "GC_ACCESS_BIFROST_PATRON_SUPPORT_ENDPOINT", "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE", diff --git a/tests/recipe-secrets.test.ts b/tests/recipe-secrets.test.ts new file mode 100644 index 0000000..974102e --- /dev/null +++ b/tests/recipe-secrets.test.ts @@ -0,0 +1,78 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { appProfiles } from "../src/app-profiles.js"; +import { providers } from "../src/contracts.js"; +import { providerCatalog } from "../src/providers.js"; + +// The Idunn recipe decides which environment names a binding may set, and +// Idunn refuses any other. So the recipe is where "a secret is only ever a +// file" is enforced for a deployment: a plaintext secret name declared here +// can carry the secret into the binding, the unit and every plan. + +const recipe = readFileSync(new URL("../deployment/idunn/recipe.toml", import.meta.url), "utf8"); + +/** The names in one `KEY = [ "A", "B" ]` array of the given TOML text. */ +function stringArray(text: string, key: string): string[] { + const match = new RegExp(`^${key}\\s*=\\s*\\[([^\\]]*)\\]`, "m").exec(text); + return match ? [...match[1]!.matchAll(/"([^"]+)"/g)].map((name) => name[1]!) : []; +} + +const serviceSection = /^\[service\]\n([\s\S]*?)(?=^\[)/m.exec(recipe.replace(/\r\n/g, "\n"))?.[1] ?? ""; +const serviceNames = [ + ...stringArray(serviceSection, "optional_environment"), + ...stringArray(serviceSection, "required_environment"), +]; +const everyDeclaredName = [...recipe.replace(/\r\n/g, "\n").matchAll(/^\w+_environment\s*=\s*\[([^\]]*)\]/gm)].flatMap( + (match) => [...match[1]!.matchAll(/"([^"]+)"/g)].map((name) => name[1]!) +); + +const serviceSecretNames = [ + "GC_ACCESS_DATABASE_URL", + "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64", + "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET", +]; + +/** Secrets Heimdall reads through readSecretInput, by their plaintext name. */ +const secretNames = [ + ...serviceSecretNames, + ...providers.map((provider) => `GC_ACCESS_PROVIDER_${provider.toUpperCase()}_CLIENT_SECRET`), + ...Object.values(appProfiles).map((profile) => `GC_ACCESS_APP_${profile.slug.toUpperCase()}_SHARED_SECRET`), +]; + +const secretShaped = /SECRET|PASSWORD|DATABASE_URL|ENCRYPTION_KEY|_PEM$/; + +describe("Idunn recipe secret declarations", () => { + it("reads the service's declared environment", () => { + expect(serviceNames).toContain("GC_ACCESS_DATABASE_URL_FILE"); + expect(serviceNames).toContain("GC_ACCESS_BASE_URL"); + }); + + it("declares no secret-shaped name except as a _FILE credential path", () => { + const plaintext = everyDeclaredName.filter((name) => secretShaped.test(name) && !name.endsWith("_FILE")); + expect(plaintext).toEqual([]); + }); + + it("declares none of the secrets Heimdall reads under its plaintext name", () => { + for (const name of secretNames) { + expect(everyDeclaredName).not.toContain(name); + } + }); + + it("declares every provider client-secret file the witness publishes", () => { + for (const provider of providers) { + expect(serviceNames).toContain(providerCatalog[provider].clientSecretEnv); + } + }); + + it("declares the shared-secret file of every app profile the witness publishes", () => { + for (const profile of Object.values(appProfiles)) { + expect(serviceNames).toContain(`GC_ACCESS_APP_${profile.slug.toUpperCase()}_SHARED_SECRET_FILE`); + } + }); + + it("declares the database URL, token key and patron secret files", () => { + for (const name of serviceSecretNames) { + expect(serviceNames).toContain(`${name}_FILE`); + } + }); +}); From 83d2f48256caf23d24b7abfc5e047703887d4ca9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:40:21 +0200 Subject: [PATCH 11/85] Docs: secrets are _FILE names; drop the unread Spotiverse secret Soul S4: the Bifrost handoff notes, the Bifrost profile and the service contract still named plaintext secret variables; each now names the _FILE credential path. .env.example and the Spotiverse profile taught GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET_FILE, which nothing reads: spotiverse is not in appSlugs. Both lines are removed rather than adding the slug. Co-Authored-By: Claude Opus 5.5 --- .env.example | 1 - docs/app-profiles/bifrost.md | 9 +++++---- docs/app-profiles/spotiverse.md | 5 ++--- docs/service-contract.md | 5 +++-- notes/fresh-workspace-handoff.md | 5 +++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index 999a299..6af5d90 100644 --- a/.env.example +++ b/.env.example @@ -30,7 +30,6 @@ GC_ACCESS_PROVIDER_YOUTUBE_CLIENT_ID= GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID= # GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE=/absolute/path/to/credential -# GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_APP_SPOTIVERSE_BACKEND_CALLBACK_URLS= # GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET_FILE=/absolute/path/to/credential GC_ACCESS_APP_GHOSTLIGHT_RUNTIME_IDS=yggdrasil-ghostlight diff --git a/docs/app-profiles/bifrost.md b/docs/app-profiles/bifrost.md index 99d37b6..976452f 100644 --- a/docs/app-profiles/bifrost.md +++ b/docs/app-profiles/bifrost.md @@ -68,10 +68,11 @@ paid membership for the requested tier title, and POSTs a signed `RecurringSupportSnapshot` fact to Bifrost's `/heimdall/patron-support/events` endpoint. -The outbound fact uses `X-Heimdall-Signature-256` with the shared -`GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET`. Bifrost verifies that HMAC, resolves -the Heimdall account id to its local user account, deduplicates by provider -event id, records the support event, and derives patron points locally. +The outbound fact uses `X-Heimdall-Signature-256` with the shared secret +Heimdall reads from `GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE`. Bifrost +verifies that HMAC, resolves the Heimdall account id to its local user account, +deduplicates by provider event id, records the support event, and derives +patron points locally. This route is a bridge from auth-owned Patreon evidence to Bifrost-owned governance meaning. Heimdall still does not own points, votes, tiers, ledger diff --git a/docs/app-profiles/spotiverse.md b/docs/app-profiles/spotiverse.md index 64a71eb..a10c72a 100644 --- a/docs/app-profiles/spotiverse.md +++ b/docs/app-profiles/spotiverse.md @@ -81,12 +81,11 @@ Heimdall needs: ```text GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID=... GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE=/absolute/path/to/spotify-client-secret -GC_ACCESS_APP_SPOTIVERSE_SHARED_SECRET_FILE=/absolute/path/to/spotiverse-shared-secret GC_ACCESS_APP_SPOTIVERSE_BACKEND_CALLBACK_URLS=https://your-spotiverse-portal.example/auth/heimdall/callback ``` -Spotiverse needs only the Heimdall base URL, matching app secret, and local -portal callback/return URLs. The callback URL Spotiverse sends in +Spotiverse needs only the Heimdall base URL and local portal callback/return +URLs. The callback URL Spotiverse sends in `HEIMDALL_CALLBACK_URL` must exactly match one URL in Heimdall's `GC_ACCESS_APP_SPOTIVERSE_BACKEND_CALLBACK_URLS` allowlist unless both services are running on the same local host. diff --git a/docs/service-contract.md b/docs/service-contract.md index 68eb6a5..c639178 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -451,8 +451,9 @@ Purpose: Authentication: - requires `x-heimdall-app-secret` -- the configured secret comes from `GC_ACCESS_APP_{APP_SLUG}_SHARED_SECRET`, - per app; there is no generic fallback secret shared across apps +- the configured secret comes from the credential file named by + `GC_ACCESS_APP_{APP_SLUG}_SHARED_SECRET_FILE`, per app; there is no generic + fallback secret shared across apps Request body: diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 9a4fdab..a8502a3 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -233,9 +233,10 @@ Bifrost needs browser verification after deployment: - configure Heimdall with `GC_ACCESS_APP_BIFROST_BACKEND_CALLBACK_URLS=https://bifrost.gamecult.org/auth/heimdall/callback` -- configure Heimdall with `GC_ACCESS_APP_BIFROST_SHARED_SECRET`, +- configure Heimdall with `GC_ACCESS_APP_BIFROST_SHARED_SECRET_FILE`, `GC_ACCESS_BIFROST_PATRON_SUPPORT_ENDPOINT=https://bifrost.gamecult.org/heimdall/patron-support/events`, - and `GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET` + and `GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE`, each naming a credential + file - configure Bifrost with `Heimdall__DiscordGuildId`, `Heimdall__DiscordAllowedRoleIds__0=`, and `Heimdall__PatreonTierTitle=Inner Sanctum` From 376dfbd2c25ca26b08caefa9a6ac18395a6f1d37 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:46:22 +0200 Subject: [PATCH 12/85] Startup database failure names the input and a code, not the URL's parts The entrypoint harness showed pg printing a piece of the database URL file on a failed startup: `getaddrinfo ENOTFOUND ` for an unresolvable host. A password with an unescaped delimiter moves part of itself into the host or the database name, and the resolver or server error then prints it. The only thing keeping a malformed URL out of stderr was pg dropping Node's `input` property (Soul S5). createStore now turns a schema-preparation failure into one error naming GC_ACCESS_DATABASE_URL_FILE and the error code, with no cause. The harness cases expect that message, and a new case puts the canary in the host. Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 13 ++++++++++++- tests/entrypoint-secrets.test.ts | 15 +++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/store/index.ts b/src/store/index.ts index 7617594..90ffd79 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -14,7 +14,18 @@ export async function createStore(config: HeimdallConfig): Promise ({ GC_ACCESS_DATABASE_URL_FILE: file("db-port", `postgres://heimdall:${CANARY}port@db:notaport/heimdall\n`) }), - expect: /./, + expect: /Postgres storage could not be prepared \(ERR_INVALID_URL\)/, }, { name: "a database URL that does not parse", env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-parse", `postgres://heimdall:${CANARY} parse@[bad/heimdall\n`) }), - expect: /./, + expect: /Postgres storage could not be prepared \(E[A-Z_]+\)/, }, { name: "a database URL with a broken percent escape", env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-percent", `postgres://heimdall:${CANARY}%zz@127.0.0.1:1/heimdall\n`) }), - expect: /./, + expect: /Postgres storage could not be prepared \(ECONNREFUSED\)/, }, { name: "a database that refuses the connection", env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-refused", `postgres://heimdall:${CANARY}refused@127.0.0.1:1/heimdall\n`) }), - expect: /ECONNREFUSED/, + expect: /Postgres storage could not be prepared \(ECONNREFUSED\)/, + }, + { + // A password holding an unescaped delimiter moves part of it into the + // host, and the resolver's error names the host. + name: "a database host that does not resolve", + env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-host", `postgres://heimdall:pw@${CANARY}host.invalid/heimdall\n`) }), + expect: /Postgres storage could not be prepared \(E[A-Z_]+\)/, }, { name: "private key text bound to the signing key path", From 33afcbf3648daaaa4131ec1a24f7557d682ec9d3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:06 +0200 Subject: [PATCH 13/85] HTTP error answers carry fixed text, never an error's message Soul pass 3 (R1): with no error handler, Fastify answered a 5xx with the error's message. A store failure under a misbound database URL reached an anonymous caller as `getaddrinfo ENOTFOUND ` or `database "" does not exist`, and a misbound URL puts part of the password in either. The same default quoted the request body in a JSON parse error (Node's JSON.parse message carries the input), the Content-Type header in a 415, and the path in a 404. buildApp now sets both handlers. A 5xx answers `{ error: "internal_error" }` and writes one stderr line naming the method, the route pattern and the error code. A schema failure answers the failed rules by schema path and Ajv message, which come from the route's schema. Any other 4xx answers `invalid_request` with the error code. An unknown route answers `not_found`. The app's own answers had two more echoes: `unknown_app` quoted the slug, and a failed provider callback put the caught error's text in the browser page and the backend handoff (the JSON answer was already fixed). Both are fixed text now; the full callback error stays in the audit event. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 49 +++++++++++--- tests/app.test.ts | 169 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 11 deletions(-) diff --git a/src/app.ts b/src/app.ts index 6f85979..322c6e8 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,6 +1,6 @@ import { createHmac, randomUUID } from "node:crypto"; import cors from "@fastify/cors"; -import Fastify, { type FastifyInstance } from "fastify"; +import Fastify, { type FastifyError, type FastifyInstance } from "fastify"; import { type AppCaller, isAppCaller, resolveAppCaller } from "./app-caller.js"; import { deliverBackendHandoff, type BackendHandoffPayload } from "./backend-handoff.js"; import { @@ -532,7 +532,7 @@ export async function startOAuthFlow( const { config, keys } = ctx; const profile = getAppProfile(input.appSlug); if (!profile) { - return { statusCode: 404, body: { error: "unknown_app", detail: `No app '${input.appSlug}' is registered.` } }; + return { statusCode: 404, body: { error: "unknown_app", detail: "No app is registered under that slug." } }; } if (!supportsProvider(profile, provider)) { @@ -736,6 +736,31 @@ export async function buildApp(options: BuildAppOptions = {}): Promise { + const status = error.statusCode !== undefined && error.statusCode >= 400 && error.statusCode < 600 ? error.statusCode : 500; + const code = typeof error.code === "string" ? error.code : "error"; + reply.code(status); + if (status >= 500) { + console.error(`Heimdall ${request.method} ${request.routeOptions.url ?? "(no route)"} failed (${code}).`); + return { error: "internal_error" }; + } + if (error.validation) { + // Ajv's messages and schema paths come from the route's schema, not the request. + const failures = error.validation.map((failure) => `${failure.schemaPath} ${failure.message ?? failure.keyword}`); + return { error: "invalid_request", detail: `${error.validationContext ?? "request"}: ${failures.join("; ")}` }; + } + return { error: "invalid_request", code }; + }); + app.setNotFoundHandler((_request, reply) => { + reply.code(404); + return { error: "not_found" }; + }); app.decorate("heimdallContext", { config, keys, store, oauthRuntimes, tokenCustody }); app.addHook("onClose", async () => { await store.close(); @@ -788,7 +813,7 @@ export async function buildApp(options: BuildAppOptions = {}): Promise { expect(tokenSet.scope).toEqual(["user:read:email"]); }); }); + +// An answer never carries an input value. The canary stands in for whatever a +// failure could quote: part of a misbound database URL in a pg or resolver +// error, a request body in a JSON parse error, a header or a path. +describe("error answers never echo an input", () => { + const CANARY = "CANARY"; + + function failingStore(): InMemoryStore { + const store = new InMemoryStore(); + store.consumeAuthCompletion = async () => { + throw Object.assign(new Error(`getaddrinfo ENOTFOUND ${CANARY}host.invalid`), { + code: "ENOTFOUND", + hostname: `${CANARY}host.invalid`, + }); + }; + return store; + } + + it("answers a store failure with a fixed 500 body and logs only the route and code", async () => { + const app = await buildApp({ config: createTestConfig(), store: failingStore() }); + apps.push(app); + const logged: string[] = []; + const originalError = console.error; + console.error = (...args: unknown[]) => void logged.push(args.map(String).join(" ")); + try { + const response = await app.inject({ + method: "POST", + url: "/v1/apps/bifrost/auth-completions/redeem", + payload: { completionCode: "x" }, + }); + + expect(response.statusCode).toBe(500); + expect(response.json()).toEqual({ error: "internal_error" }); + } finally { + console.error = originalError; + } + expect(logged).toEqual(["Heimdall POST /v1/apps/:appSlug/auth-completions/redeem failed (ENOTFOUND)."]); + }); + + it("answers a body that is not JSON without quoting it", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + + const response = await app.inject({ + method: "POST", + url: "/v1/apps/bifrost/auth-completions/redeem", + headers: { "content-type": "application/json" }, + payload: `${CANARY}notjson`, + }); + + expect(response.statusCode).toBe(400); + expect(response.json().error).toBe("invalid_request"); + expect(response.body).not.toContain(CANARY); + }); + + it("answers an unsupported media type without quoting the header", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + + const response = await app.inject({ + method: "POST", + url: "/v1/apps/bifrost/auth-completions/redeem", + headers: { "content-type": `application/${CANARY}` }, + payload: "x", + }); + + expect(response.statusCode).toBe(415); + expect(response.body).not.toContain(CANARY); + }); + + it("names the failed schema rule without quoting the request", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + + const response = await app.inject({ + method: "POST", + url: "/v1/apps/bifrost/auth-completions/redeem", + payload: { [`${CANARY}extra`]: `${CANARY}value` }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json().error).toBe("invalid_request"); + expect(response.json().detail).toMatch(/^body: #\//); + expect(response.body).not.toContain(CANARY); + }); + + it("answers an unknown route or app without quoting the path", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + + const route = await app.inject({ method: "GET", url: `/v1/${CANARY}route` }); + expect(route.statusCode).toBe(404); + expect(route.json()).toEqual({ error: "not_found" }); + + const unknownApp = await app.inject({ method: "GET", url: `/v1/apps/${CANARY}app` }); + expect(unknownApp.statusCode).toBe(404); + expect(unknownApp.json().error).toBe("unknown_app"); + expect(unknownApp.body).not.toContain(CANARY); + + const start = await app.inject({ + method: "POST", + url: "/v1/oauth/discord/start", + payload: { appSlug: `${CANARY}app`, mode: "sign_in", returnTo: "https://repixelizer.gamecult.org/app/" }, + }); + expect(start.statusCode).toBe(400); + expect(start.body).not.toContain(CANARY); + }); + + function failingDiscordRuntime(): OAuthProviderRuntime { + return { + ...createMockDiscordRuntime(), + async exchangeAuthorizationCode() { + throw new SyntaxError(`Unexpected token 'a', "access_token=${CANARY}token" is not valid JSON`); + }, + }; + } + + it("renders a failed provider callback page without the failure's text", async () => { + const app = await buildApp({ config: createTestConfig(), oauthRuntimes: { discord: failingDiscordRuntime() } }); + apps.push(app); + + const stateToken = await startDiscordSignIn(app); + const response = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=test-code&state=${encodeURIComponent(stateToken)}`, + headers: { accept: "text/html" }, + }); + + expect(response.headers["content-type"]).toContain("text/html"); + expect(response.body).toContain("oauth_callback_failed"); + expect(response.body).not.toContain(CANARY); + }); + + it("delivers a failed provider callback to the app backend without the failure's text", async () => { + const deliveries: string[] = []; + globalThis.fetch = async (_input, init) => { + deliveries.push(String(init?.body)); + return new Response(null, { status: 204 }); + }; + const app = await buildApp({ config: createTestConfig(), oauthRuntimes: { discord: failingDiscordRuntime() } }); + apps.push(app); + + const start = await app.inject({ + method: "POST", + url: "/v1/oauth/discord/start", + payload: { + appSlug: "repixelizer", + mode: "sign_in", + returnTo: "https://repixelizer.gamecult.org/app/", + handoff: { + kind: "backend_callback", + attemptId: "attempt-123", + callbackUrl: "https://repixelizer.gamecult.org/api/auth/heimdall/callback", + }, + }, + }); + expect(start.statusCode).toBe(201); + const response = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=test-code&state=${encodeURIComponent(start.json().stateToken as string)}`, + }); + + expect(response.statusCode).toBe(502); + expect(response.body).not.toContain(CANARY); + expect(deliveries).toHaveLength(1); + expect(JSON.parse(deliveries[0]!)).toMatchObject({ status: "error", error: "oauth_callback_failed" }); + expect(deliveries[0]).not.toContain(CANARY); + }); +}); From 0f5ffd9e697543d191d89d2ad08dfbf4cd226ee2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:10 +0200 Subject: [PATCH 14/85] Every start connects to Postgres; pin the signing key's read refusal Soul pass 3 (R1, R4). With GC_ACCESS_APPLY_SCHEMA_ON_STARTUP=0, startup never opened a connection, so a misbound database URL surfaced only in a request's error answer. createStore now connects on every Postgres start: it applies the schema when asked and otherwise runs `SELECT 1`, and either failure is the same fixed message naming GC_ACCESS_DATABASE_URL_FILE and a code. The entrypoint harness gains that case with the canary in the host. The harness also pins the signing key's read-failure branch with a key file of mode 000. Root reads such a file, so the case runs the entrypoint as a user a mode can refuse: the test process itself when it is not root, or else the checkout's owner. It is skipped, with the reason in its title, on Windows or as root in a root-owned checkout. Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 34 +++++++++--------- src/store/postgres.ts | 7 +++- tests/entrypoint-secrets.test.ts | 60 +++++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/src/store/index.ts b/src/store/index.ts index 90ffd79..d4d2ab5 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -8,24 +8,24 @@ export { InMemoryStore } from "./in-memory.js"; export { PostgresStore, createPostgresStore } from "./postgres.js"; export async function createStore(config: HeimdallConfig): Promise { - const store = - config.storage.backend === "postgres" - ? createPostgresStore(config.storage.databaseUrl ?? "postgres://127.0.0.1/heimdall") - : new InMemoryStore(); + if (config.storage.backend !== "postgres") { + return new InMemoryStore(); + } - if (config.storage.applySchemaOnStartup) { - try { - await store.ensureSchema(); - } catch (error) { - // pg's and Node's errors name the host, the database or the URL itself, - // and any of those can hold part of a secret that was bound or escaped - // wrongly. Startup prints this error, so it names only the input and a - // code, and carries no cause. - const code = (error as { code?: unknown }).code; - throw new Error( - `Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); check GC_ACCESS_DATABASE_URL_FILE.` - ); - } + const store = createPostgresStore(config.storage.databaseUrl ?? "postgres://127.0.0.1/heimdall"); + try { + // Connect on every start, whether or not it applies the schema, so a + // misbound URL fails here rather than in the first request. + await (config.storage.applySchemaOnStartup ? store.ensureSchema() : store.checkConnection()); + } catch (error) { + // pg's and Node's errors name the host, the database or the URL itself, + // and any of those can hold part of a secret that was bound or escaped + // wrongly. Startup prints this error, so it names only the input and a + // code, and carries no cause. + const code = (error as { code?: unknown }).code; + throw new Error( + `Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); check GC_ACCESS_DATABASE_URL_FILE.` + ); } return store; diff --git a/src/store/postgres.ts b/src/store/postgres.ts index a9c376f..3ea5f88 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -291,6 +291,11 @@ export class PostgresStore implements HeimdallStore { await this.pool.query(CREATE_SCHEMA_SQL); } + /** Opens a connection and runs a trivial query, so a start that does not apply the schema still proves the URL. */ + async checkConnection(): Promise { + await this.pool.query("SELECT 1"); + } + async close(): Promise { await this.pool.end(); } @@ -791,7 +796,7 @@ export class PostgresStore implements HeimdallStore { } } -export function createPostgresStore(databaseUrl: string): HeimdallStore { +export function createPostgresStore(databaseUrl: string): PostgresStore { const pool = new Pool({ connectionString: databaseUrl, }); diff --git a/tests/entrypoint-secrets.test.ts b/tests/entrypoint-secrets.test.ts index c52c2ef..941c8c5 100644 --- a/tests/entrypoint-secrets.test.ts +++ b/tests/entrypoint-secrets.test.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -35,11 +35,31 @@ interface Run { output: string; } +interface Identity { + uid: number; + gid: number; +} + +/** + * Who can be denied a read. Root reads a mode-000 file, so a read failure is + * only reachable as another user: the process itself when it is not root, or + * else the checkout's owner, who can still load the entrypoint. A string is + * the reason no such user exists here. + */ +function nonRootReader(): Identity | "self" | string { + if (process.platform === "win32") return "file modes do not deny reads on Windows"; + if (process.getuid?.() !== 0) return "self"; + const owner = statSync(repoRoot); + if (owner.uid === 0) return "running as root in a root-owned checkout, so no other user can load the entrypoint"; + return { uid: owner.uid, gid: owner.gid }; +} + /** Runs `tsx src/index.ts` with only the given environment, merging stdout and stderr. */ -function runEntrypoint(name: string, env: Record): Promise { +function runEntrypoint(name: string, env: Record, identity?: Identity): Promise { return new Promise((resolve, reject) => { const child = spawn(process.execPath, [tsxCli, "src/index.ts"], { cwd: repoRoot, + ...identity, env: { PATH: process.env.PATH ?? "", HOME: scratch, @@ -73,6 +93,8 @@ interface Case { env: () => Record; /** Something the failure must say, so a case cannot pass by never reaching its path. */ expect: RegExp; + /** Run the entrypoint as a user a file mode can refuse. */ + nonRoot?: true; } const cases: Case[] = [ @@ -136,6 +158,16 @@ const cases: Case[] = [ env: () => ({ GC_ACCESS_DATABASE_URL_FILE: file("db-host", `postgres://heimdall:pw@${CANARY}host.invalid/heimdall\n`) }), expect: /Postgres storage could not be prepared \(E[A-Z_]+\)/, }, + { + // A start that does not apply the schema still connects, so a misbound URL + // fails at startup rather than in a request's error answer. + name: "a database host that does not resolve, on a start that does not apply the schema", + env: () => ({ + GC_ACCESS_APPLY_SCHEMA_ON_STARTUP: "0", + GC_ACCESS_DATABASE_URL_FILE: file("db-host-noapply", `postgres://heimdall:pw@${CANARY}host.invalid/heimdall\n`), + }), + expect: /Postgres storage could not be prepared \(E[A-Z_]+\)/, + }, { name: "private key text bound to the signing key path", env: () => ({ @@ -153,6 +185,19 @@ const cases: Case[] = [ env: () => ({ GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: file("not-a-key", `${CANARY}notakey\n`) }), expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH does not hold a usable private key/, }, + { + name: "a signing key file its reader may not read", + env: () => { + const key = file(`${CANARY}locked-key`, `${CANARY}pem\n`); + chmodSync(key, 0o000); + const dataRoot = join(scratch, "locked-key-data"); + mkdirSync(dataRoot, { recursive: true }); + chmodSync(dataRoot, 0o777); + return { GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: key, GC_ACCESS_DATA_ROOT: dataRoot }; + }, + expect: /GC_ACCESS_SIGNING_PRIVATE_KEY_PATH could not be read \(EACCES\)/, + nonRoot: true, + }, { name: "a signing key path that is a directory", env: () => { @@ -187,10 +232,17 @@ const cases: Case[] = [ describe("the entrypoint never prints a secret", () => { writeFileSync(join(scratch, "blank"), " \n"); + // mkdtemp makes the scratch directory private to its creator; a non-root + // reader must still reach the files a case names inside it. + chmodSync(scratch, 0o755); + const reader = nonRootReader(); for (const testCase of cases) { - it(`refuses ${testCase.name} without printing the canary`, async () => { - const run = await runEntrypoint(testCase.name.replace(/\W+/g, "-"), testCase.env()); + const skipReason = testCase.nonRoot && typeof reader === "string" && reader !== "self" ? reader : undefined; + const title = `refuses ${testCase.name} without printing the canary${skipReason ? ` (skipped: ${skipReason})` : ""}`; + it.skipIf(skipReason !== undefined)(title, async () => { + const identity = testCase.nonRoot && typeof reader === "object" ? reader : undefined; + const run = await runEntrypoint(testCase.name.replace(/\W+/g, "-"), testCase.env(), identity); expect(run.signal, "the process must stop by itself").toBeNull(); expect(run.code, "startup must fail").not.toBe(0); From c626a86d55614d09d35201d1459f4541c77d5730 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:15 +0200 Subject: [PATCH 15/85] A lost idle Postgres connection no longer kills Heimdall Soul pass 3 (R2): the pool had no 'error' listener. pg emits an idle client's connection loss there (a database restart, a terminated backend), so the event went unheard, the process died, and Node printed the pg Client with its connection parameters. createPostgresStore now listens and writes one line naming the error code. The pool has already dropped the dead client and connects again on the next query. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 12 ++++++++++++ tests/store.test.ts | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 3ea5f88..0ae0ea5 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -800,5 +800,17 @@ export function createPostgresStore(databaseUrl: string): PostgresStore { const pool = new Pool({ connectionString: databaseUrl, }); + // An idle client whose connection ends (a database restart, an + // administrator terminating the backend) is emitted here. Unheard, the + // event kills the process and Node prints the pg Client, connection + // parameters included. The pool has already dropped the dead client and + // opens a new one on the next query, so this only says what happened, by + // code. + pool.on("error", (error) => { + const code = (error as { code?: unknown }).code; + console.error( + `Heimdall lost an idle Postgres connection (${typeof code === "string" ? code : "error"}); the pool reconnects on the next query.` + ); + }); return new PostgresStore(pool); } diff --git a/tests/store.test.ts b/tests/store.test.ts index e8db032..30d6861 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { newDb } from "pg-mem"; -import { PostgresStore } from "../src/store/postgres.js"; +import { createPostgresStore, PostgresStore } from "../src/store/postgres.js"; describe("PostgresStore", () => { it("round-trips core auth records through postgres", async () => { @@ -183,3 +183,29 @@ describe("PostgresStore", () => { await store.close(); }); }); + +describe("createPostgresStore", () => { + // pg emits an idle client's connection loss on the pool. With no listener + // that event kills the process, and Node prints the pg Client with its + // connection parameters. + it("survives an idle connection's loss and says so by code alone", async () => { + const store = createPostgresStore("postgres://heimdall:CANARYpw@127.0.0.1:1/heimdall"); + const pool = (store as unknown as { pool: { emit(event: string, ...args: unknown[]): boolean } }).pool; + const logged: string[] = []; + const originalError = console.error; + console.error = (...args: unknown[]) => void logged.push(args.map(String).join(" ")); + try { + const terminated = Object.assign(new Error("terminating connection due to administrator command CANARY"), { + code: "57P01", + client: { connectionParameters: { password: "CANARYpw" } }, + }); + expect(() => pool.emit("error", terminated, terminated.client)).not.toThrow(); + } finally { + console.error = originalError; + await store.close(); + } + expect(logged).toEqual([ + "Heimdall lost an idle Postgres connection (57P01); the pool reconnects on the next query.", + ]); + }); +}); From ca2e78f00c955630214cd43fc14c837855f898ef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:30:18 +0200 Subject: [PATCH 16/85] Recipe test reads its secret names from the source Soul pass 3 (R3): the recipe test's list of secret names was written by hand, and its secret-shaped pattern missed TOKEN, API_KEY, PRIVATE_KEY, SALT, CREDENTIAL and DSN. A new secret could be read in the source and declared in plaintext in the recipe without a failure. The test now takes the names from every readSecretInput call under src/: a literal as written, the two computed names over the lists that compute them, and a failure for any argument it cannot expand. Every one of those must be declared as its _FILE and never by its plaintext name. The pattern is wider, with two named exemptions: the signing key's path and its bootstrap flag. Co-Authored-By: Claude Opus 5.5 --- tests/recipe-secrets.test.ts | 71 ++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/tests/recipe-secrets.test.ts b/tests/recipe-secrets.test.ts index 974102e..f184e63 100644 --- a/tests/recipe-secrets.test.ts +++ b/tests/recipe-secrets.test.ts @@ -1,7 +1,7 @@ -import { readFileSync } from "node:fs"; +import { readdirSync, readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { appProfiles } from "../src/app-profiles.js"; -import { providers } from "../src/contracts.js"; +import { appSlugs, providers } from "../src/contracts.js"; import { providerCatalog } from "../src/providers.js"; // The Idunn recipe decides which environment names a binding may set, and @@ -26,20 +26,45 @@ const everyDeclaredName = [...recipe.replace(/\r\n/g, "\n").matchAll(/^\w+_envir (match) => [...match[1]!.matchAll(/"([^"]+)"/g)].map((name) => name[1]!) ); -const serviceSecretNames = [ - "GC_ACCESS_DATABASE_URL", - "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64", - "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET", -]; +// The secrets Heimdall reads are whatever its source passes to +// readSecretInput, so the set is read from the source: a secret added there +// cannot be declared in plaintext here without failing. A literal name is +// taken as written; the two computed names expand over the lists that compute +// them; any other argument fails, because this test cannot know its names. +const srcRoot = new URL("../src/", import.meta.url); +const secretArguments = readdirSync(srcRoot, { recursive: true }) + .map(String) + .filter((file) => file.endsWith(".ts")) + .flatMap((file) => + [...readFileSync(new URL(file.replace(/\\/g, "/"), srcRoot), "utf8").matchAll(/(? call[1]! + ) + ); + +const computedSecretNames: Record = { + "`${prefix}_CLIENT_SECRET`": providers.map((provider) => `GC_ACCESS_PROVIDER_${provider.toUpperCase()}_CLIENT_SECRET`), + envKey: appSlugs.map((appSlug) => `GC_ACCESS_APP_${appSlug.toUpperCase()}_SHARED_SECRET`), +}; /** Secrets Heimdall reads through readSecretInput, by their plaintext name. */ -const secretNames = [ - ...serviceSecretNames, - ...providers.map((provider) => `GC_ACCESS_PROVIDER_${provider.toUpperCase()}_CLIENT_SECRET`), - ...Object.values(appProfiles).map((profile) => `GC_ACCESS_APP_${profile.slug.toUpperCase()}_SHARED_SECRET`), -]; +const secretNames = secretArguments.flatMap((argument) => { + const literal = /^"([A-Z0-9_]+)"$/.exec(argument); + if (literal) return [literal[1]!]; + const computed = computedSecretNames[argument]; + if (computed) return computed; + throw new Error(`recipe-secrets cannot derive the names read by readSecretInput(env, ${argument}); teach it.`); +}); -const secretShaped = /SECRET|PASSWORD|DATABASE_URL|ENCRYPTION_KEY|_PEM$/; +const secretShaped = + /SECRET|PASSWORD|PASSPHRASE|TOKEN|API_KEY|PRIVATE_KEY|ENCRYPTION_KEY|SALT|CREDENTIAL|DATABASE_URL|_DSN|_PEM/; + +/** Secret-shaped names that hold no secret, each for its own reason. */ +const notSecrets = new Set([ + // A path to the signing key file, which Heimdall reads (or creates) itself. + "GC_ACCESS_SIGNING_PRIVATE_KEY_PATH", + // A flag: create that file on first boot. + "GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP", +]); describe("Idunn recipe secret declarations", () => { it("reads the service's declared environment", () => { @@ -48,10 +73,24 @@ describe("Idunn recipe secret declarations", () => { }); it("declares no secret-shaped name except as a _FILE credential path", () => { - const plaintext = everyDeclaredName.filter((name) => secretShaped.test(name) && !name.endsWith("_FILE")); + const plaintext = everyDeclaredName.filter( + (name) => secretShaped.test(name) && !name.endsWith("_FILE") && !notSecrets.has(name) + ); expect(plaintext).toEqual([]); }); + it("finds every readSecretInput call in the source", () => { + expect(secretNames).toEqual( + expect.arrayContaining([ + "GC_ACCESS_DATABASE_URL", + "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64", + "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET", + "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET", + "GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET", + ]) + ); + }); + it("declares none of the secrets Heimdall reads under its plaintext name", () => { for (const name of secretNames) { expect(everyDeclaredName).not.toContain(name); @@ -70,8 +109,8 @@ describe("Idunn recipe secret declarations", () => { } }); - it("declares the database URL, token key and patron secret files", () => { - for (const name of serviceSecretNames) { + it("declares the _FILE of every secret Heimdall reads", () => { + for (const name of secretNames) { expect(serviceNames).toContain(`${name}_FILE`); } }); From 869802126cd9b2cc206595d739568d317e94f971 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:37:11 +0200 Subject: [PATCH 17/85] Pin the 4xx branch of the error handler with a thrown error A mutant that put the message back into the non-validation 4xx answer survived: Fastify 5.8.5's own 4xx errors (invalid or empty JSON body, 415, 413) carry fixed messages, so the JSON-parse and media-type cases could not tell. The branch protects against any error thrown with a 4xx status, whose message is whatever its thrower wrote. A test route now throws one with the canary, and the answer must be the code alone. The handler's comment and the earlier commit's message said JSON.parse and the 415 quote input; in this Fastify they do not. The comment is corrected. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 12 ++++++------ tests/app.test.ts | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/app.ts b/src/app.ts index 322c6e8..f6a72e9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -736,12 +736,12 @@ export async function buildApp(options: BuildAppOptions = {}): Promise { const status = error.statusCode !== undefined && error.statusCode >= 400 && error.statusCode < 600 ? error.statusCode : 500; const code = typeof error.code === "string" ? error.code : "error"; diff --git a/tests/app.test.ts b/tests/app.test.ts index 79c562c..de2a896 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1224,7 +1224,7 @@ describe("Heimdall service", () => { // An answer never carries an input value. The canary stands in for whatever a // failure could quote: part of a misbound database URL in a pg or resolver -// error, a request body in a JSON parse error, a header or a path. +// error, a request body, a header, a path, or a 4xx error's own message. describe("error answers never echo an input", () => { const CANARY = "CANARY"; @@ -1276,6 +1276,19 @@ describe("error answers never echo an input", () => { expect(response.body).not.toContain(CANARY); }); + it("answers any other 4xx error by its code, not its message", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + app.get("/v1/probe-4xx", async () => { + throw Object.assign(new Error(`refused ${CANARY}input`), { statusCode: 409, code: "PROBE_REFUSED" }); + }); + + const response = await app.inject({ method: "GET", url: "/v1/probe-4xx" }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toEqual({ error: "invalid_request", code: "PROBE_REFUSED" }); + }); + it("answers an unsupported media type without quoting the header", async () => { const app = await buildApp({ config: createTestConfig() }); apps.push(app); From cad12165a9218e1dbf6d415dadecb10e288db15d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:49:23 +0200 Subject: [PATCH 18/85] A start that does not apply the schema proves the schema is there Self's ruling on the open apply=0 item: connecting proves the URL, not the schema, so a good URL to an empty database started and then answered every request with a 500 (42P01). PostgresStore.checkSchema replaces checkConnection. One query finds every table and index CREATE_SCHEMA_SQL creates and every column it adds, and anything missing fails with SCHEMA_MISSING, which startup reports through the same fixed message. The required names are read from CREATE_SCHEMA_SQL itself, so a new table or migration is required as soon as it is written. Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 7 ++++--- src/store/postgres.ts | 33 +++++++++++++++++++++++++++++---- src/store/schema.ts | 13 +++++++++++++ tests/store.test.ts | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/src/store/index.ts b/src/store/index.ts index d4d2ab5..b281ec4 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -14,9 +14,10 @@ export async function createStore(config: HeimdallConfig): Promise { - await this.pool.query("SELECT 1"); + /** + * Proves, for a start that does not apply the schema, that the URL connects + * and that the database holds every relation and column the schema makes. + * A missing one fails with code SCHEMA_MISSING. + */ + async checkSchema(): Promise { + const result = await this.pool.query<{ missing: string }>( + ` + SELECT name AS missing FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL + UNION ALL + SELECT required.table_name || '.' || required.column_name + FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name) + WHERE NOT EXISTS ( + SELECT 1 FROM information_schema.columns c + WHERE c.table_schema = current_schema() + AND c.table_name = required.table_name + AND c.column_name = required.column_name + ) + `, + [ + REQUIRED_RELATIONS, + REQUIRED_COLUMNS.map((required) => required.table), + REQUIRED_COLUMNS.map((required) => required.column), + ] + ); + if (result.rows.length > 0) { + throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); + } } async close(): Promise { diff --git a/src/store/schema.ts b/src/store/schema.ts index 17ddc49..8e9c921 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -162,3 +162,16 @@ CREATE TABLE IF NOT EXISTS audit_events ( CREATE INDEX IF NOT EXISTS audit_events_lookup_idx ON audit_events(account_id, session_id, app_slug, created_at); `; + +/** + * What a start that does not apply the schema must find, read from the SQL + * above so the two cannot drift: every table and index it creates, and every + * column it adds to an existing table. + */ +export const REQUIRED_RELATIONS = [ + ...CREATE_SCHEMA_SQL.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), +].map((match) => match[1]!); + +export const REQUIRED_COLUMNS = [ + ...CREATE_SCHEMA_SQL.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm), +].map((match) => ({ table: match[1]!, column: match[2]! })); diff --git a/tests/store.test.ts b/tests/store.test.ts index 30d6861..c28e353 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { newDb } from "pg-mem"; import { createPostgresStore, PostgresStore } from "../src/store/postgres.js"; +import { REQUIRED_COLUMNS, REQUIRED_RELATIONS } from "../src/store/schema.js"; describe("PostgresStore", () => { it("round-trips core auth records through postgres", async () => { @@ -209,3 +210,35 @@ describe("createPostgresStore", () => { ]); }); }); + +describe("PostgresStore.checkSchema", () => { + it("requires every table and index the schema creates and every column it adds", () => { + expect(REQUIRED_RELATIONS).toEqual( + expect.arrayContaining(["accounts", "audit_events", "auth_completions_attempt_unconsumed_unique_idx", "audit_events_lookup_idx"]) + ); + expect(REQUIRED_RELATIONS).toHaveLength(17); + expect(REQUIRED_COLUMNS).toEqual([{ table: "auth_completions", column: "attempt_id" }]); + }); + + function storeAnswering(missing: string[]) { + const queries: unknown[][] = []; + const store = new PostgresStore({ + query: (async (_text: string, values: unknown[]) => { + queries.push(values); + return { rows: missing.map((name) => ({ missing: name })) }; + }) as never, + end: async () => undefined, + }); + return { store, queries }; + } + + it("refuses with SCHEMA_MISSING when anything is missing", async () => { + const { store, queries } = storeAnswering(["auth_completions.attempt_id"]); + await expect(store.checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" }); + expect(queries).toEqual([[REQUIRED_RELATIONS, ["auth_completions"], ["attempt_id"]]]); + }); + + it("passes when nothing is missing", async () => { + await expect(storeAnswering([]).store.checkSchema()).resolves.toBeUndefined(); + }); +}); From 7de12e84be92e2696f1aed7d0f8292a941f1e170 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:49:30 +0200 Subject: [PATCH 19/85] Relayed provider errors are printable, bounded and inert in the page Self's ruling: the provider_error relay stays, because it is the protocol, but its error and error_description are query parameters anyone can write. Only the characters RFC 6749 4.1.2.1 allows in them survive (%x20-21, %x23-5B, %x5D-7E), and each is cut to a bound (64 and 256). The JSON answer, the browser page and the backend handoff all take the relayed values. The page has no template engine. Every value in it goes through serializeForScript, which escapes `<`, apart from the fallback URL in the link's href, which the URL serializer percent-encodes. A test puts `"\\\u0007é${"x".repeat(300)}`; + const allowed = /^[\x20-\x21\x23-\x5b\x5d-\x7e]*$/; + + function providerErrorUrl(stateToken: string): string { + const query = new URLSearchParams({ state: stateToken, error: `${"e".repeat(100)}\u0000`, error_description: injected }); + return `/v1/oauth/discord/callback?${query.toString()}`; + } + + it("is printable and bounded in the JSON answer", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + + const response = await app.inject({ method: "GET", url: providerErrorUrl(await startDiscordSignIn(app)) }); + + expect(response.statusCode).toBe(400); + const { providerError, providerErrorDescription } = response.json() as Record; + expect(providerError).toBe("e".repeat(64)); + expect(providerErrorDescription).toHaveLength(256); + expect(providerErrorDescription).toMatch(allowed); + expect(providerErrorDescription.startsWith("xxx")).toBe(true); + }); + + it("cannot inject markup into the browser page, and is bounded there", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + + const response = await app.inject({ + method: "GET", + url: providerErrorUrl(await startDiscordSignIn(app)), + headers: { accept: "text/html" }, + }); + + expect(response.headers["content-type"]).toContain("text/html"); + expect(response.body.match(/xxx")).toBe(true); + expect(String(providerErrorDescription).startsWith("xxx")).toBe(true); }); it("cannot inject markup into the browser page, and is bounded there", async () => { From efd11373b815d3f5b57bd6ec02452b5c6f88767b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:03:57 +0200 Subject: [PATCH 22/85] Audit event keeps a provider's HTTP status, never its body Self's ruling: the failed-callback audit event lost the provider's status when it stopped storing the message. A provider's non-2xx answer is now a ProviderHttpError carrying the status, and its message no longer holds the response body, which nothing needs and which can quote the request. The audit event adds providerStatus: an integer in 100-599, or null. Tests drive the real Discord token exchange against a 401 whose body holds the canary, and throw a status of 9999. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 4 +++- src/oauth.ts | 21 ++++++++++++----- tests/app.test.ts | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/app.ts b/src/app.ts index 09d8266..670320a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1332,8 +1332,9 @@ export async function buildApp(options: BuildAppOptions = {}): Promise= 100 && status <= 599 ? status : null, }, createdAt: nowIso, }); diff --git a/src/oauth.ts b/src/oauth.ts index 564b7b8..0d3872c 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -59,8 +59,19 @@ export interface OAuthProviderRuntime { }): Promise; } -function buildErrorMessage(status: number, body: string, fallback: string): string { - return `${fallback} (status ${status}): ${body || "empty response"}`; +/** + * A provider answered with a non-2xx status. It carries the status and never + * the response body: a provider's body can quote the request, and whatever + * catches this may record or print it. + */ +export class ProviderHttpError extends Error { + readonly status: number; + + constructor(status: number, failure: string) { + super(`${failure} (status ${status}).`); + this.name = "ProviderHttpError"; + this.status = status; + } } async function fetchJson(input: string | URL, init: RequestInit, fallbackError: string): Promise { @@ -68,7 +79,7 @@ async function fetchJson(input: string | URL, init: RequestInit, fallbackErro const text = await response.text(); if (!response.ok) { - throw new Error(buildErrorMessage(response.status, text, fallbackError)); + throw new ProviderHttpError(response.status, fallbackError); } return (text ? JSON.parse(text) : {}) as T; @@ -926,10 +937,10 @@ async function evaluateDiscordEntitlements(options: { }; } - const text = await response.text(); if (!response.ok) { - throw new Error(buildErrorMessage(response.status, text, "Discord guild member lookup failed")); + throw new ProviderHttpError(response.status, "Discord guild member lookup failed"); } + const text = await response.text(); const member = (text ? JSON.parse(text) : {}) as { nick?: string; diff --git a/tests/app.test.ts b/tests/app.test.ts index 3840756..a529150 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1511,7 +1511,64 @@ describe("the failed-callback audit event", () => { mode: "sign_in", errorClass: "Error", errorCode: "ENOTFOUND", + providerStatus: null, }); expect(JSON.stringify(events)).not.toContain("CANARY"); }); }); + +describe("the failed-callback audit event for a provider's HTTP refusal", () => { + async function auditAfterTokenExchangeAnswers(response: () => Response) { + globalThis.fetch = async () => response(); + const store = new InMemoryStore(); + const app = await buildApp({ config: createTestConfig(), store, oauthRuntimes: createOAuthRuntimeRegistry() }); + apps.push(app); + const stateToken = await startDiscordSignIn(app); + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=test-code&state=${encodeURIComponent(stateToken)}`, + }); + expect(callback.statusCode).toBe(502); + const events = [...(store as unknown as { auditEvents: Map }).auditEvents.values()]; + return events.filter((event) => event.eventType === "oauth_callback_failed"); + } + + it("keeps the provider's status as a number and never its body", async () => { + const failed = await auditAfterTokenExchangeAnswers( + () => new Response('{"error":"invalid_grant","echo":"CANARYbody"}', { status: 401 }) + ); + + expect(failed).toHaveLength(1); + expect(failed[0]!.eventPayloadJson).toEqual({ + provider: "discord", + mode: "sign_in", + errorClass: "ProviderHttpError", + errorCode: null, + providerStatus: 401, + }); + expect(JSON.stringify(failed)).not.toContain("CANARY"); + }); + + it("keeps no status that is not an HTTP status", async () => { + const store = new InMemoryStore(); + const app = await buildApp({ + config: createTestConfig(), + store, + oauthRuntimes: { + discord: { + ...createMockDiscordRuntime(), + async exchangeAuthorizationCode() { + throw Object.assign(new Error("refused"), { status: 9999 }); + }, + }, + }, + }); + apps.push(app); + const stateToken = await startDiscordSignIn(app); + await app.inject({ method: "GET", url: `/v1/oauth/discord/callback?code=test-code&state=${encodeURIComponent(stateToken)}` }); + + const events = [...(store as unknown as { auditEvents: Map }> }).auditEvents.values()]; + const failed = events.find((event) => event.eventType === "oauth_callback_failed"); + expect(failed?.eventPayloadJson.providerStatus).toBeNull(); + }); +}); From 698f5dfc0ea5af4e5781fdcedd81224b068adb09 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:07:16 +0200 Subject: [PATCH 23/85] Private command diagnostics carry class and code, never a message Soul pass 4, F1: a provider's 2xx refresh answer that was not JSON made JSON.parse quote the text around the fault, a refresh-token fragment among it, and the plane returned that message in the reply's diagnostics. Those travel outside the sealed envelope and Ghostlight logs them. The plane's diagnostic is now fixed text plus the error's class and code, through errorIdentity, which the failed-callback audit event now shares. Every provider body Heimdall parses goes through readProviderJson: a non-2xx answer or a body that is not JSON becomes a ProviderResponseError (status and code, provider_status or provider_body_not_json) and no parser message survives. This replaces ProviderHttpError and the separate parses in the Discord guild lookup and the YouTube channel lookup. Soul's canary probe is committed inverted: a refresh against a malformed 2xx body and against a 400 whose body holds the canary; no byte of it may appear in the reply or its decoded failure payload. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 18 +++- src/oauth.ts | 50 ++++++---- src/private-command-plane.ts | 10 +- tests/app.test.ts | 20 +++- tests/private-command-diagnostics.test.ts | 111 ++++++++++++++++++++++ 5 files changed, 180 insertions(+), 29 deletions(-) create mode 100644 tests/private-command-diagnostics.test.ts diff --git a/src/app.ts b/src/app.ts index 670320a..a63117c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -249,6 +249,19 @@ function relayedProviderText(value: string | undefined, bound: number): string | return printable ? printable : undefined; } +/** + * What may be recorded or answered about a caught error: its class and its + * code, each only when it is a short identifier. Never its message, which + * carries whatever the thrower quoted (a host, a database, a provider's body). + */ +export function errorIdentity(error: unknown): { errorClass: string; errorCode: string | null } { + const code = (error as { code?: unknown } | null)?.code; + return { + errorClass: error instanceof Error && /^[A-Za-z]{1,64}$/.test(error.name) ? error.name : "Error", + errorCode: typeof code === "string" && /^[A-Za-z0-9_]{1,64}$/.test(code) ? code : null, + }; +} + function prefersHtml(acceptHeader: string | undefined): boolean { if (!acceptHeader) { return false; @@ -1333,9 +1346,7 @@ export async function buildApp(options: BuildAppOptions = {}): Promise= 100 && status <= 599 ? status : null, }, createdAt: nowIso, diff --git a/src/oauth.ts b/src/oauth.ts index 0d3872c..33ae495 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -60,29 +60,42 @@ export interface OAuthProviderRuntime { } /** - * A provider answered with a non-2xx status. It carries the status and never - * the response body: a provider's body can quote the request, and whatever - * catches this may record or print it. + * A provider answer Heimdall cannot use: a non-2xx status, or a body that is + * not JSON. It carries the status and a code, never the body or the parser's + * message: a provider's body can quote the request or hold tokens (JSON.parse + * quotes the text around the fault), and whatever catches this may record, + * print or relay it. */ -export class ProviderHttpError extends Error { +export class ProviderResponseError extends Error { readonly status: number; + readonly code: "provider_status" | "provider_body_not_json"; - constructor(status: number, failure: string) { - super(`${failure} (status ${status}).`); - this.name = "ProviderHttpError"; + constructor(status: number, code: ProviderResponseError["code"], failure: string) { + super(`${failure} (status ${status}, ${code}).`); + this.name = "ProviderResponseError"; this.status = status; + this.code = code; } } -async function fetchJson(input: string | URL, init: RequestInit, fallbackError: string): Promise { - const response = await fetch(input, init); +/** The JSON body of a provider's answer; refuses any other answer by status and code alone. */ +async function readProviderJson(response: Response, failure: string): Promise { const text = await response.text(); - if (!response.ok) { - throw new ProviderHttpError(response.status, fallbackError); + throw new ProviderResponseError(response.status, "provider_status", failure); } + if (!text) { + return {} as T; + } + try { + return JSON.parse(text) as T; + } catch { + throw new ProviderResponseError(response.status, "provider_body_not_json", failure); + } +} - return (text ? JSON.parse(text) : {}) as T; +async function fetchJson(input: string | URL, init: RequestInit, failure: string): Promise { + return readProviderJson(await fetch(input, init), failure); } function normalizeScopes(scope: string | string[] | undefined): string[] { @@ -783,7 +796,7 @@ async function resolveYouTubeIdentity(options: { accessToken: string }): Promise }); if (channelResponse.ok) { - const payload = (await channelResponse.json()) as { + const payload = await readProviderJson<{ items?: Array<{ id: string; snippet?: { @@ -796,7 +809,7 @@ async function resolveYouTubeIdentity(options: { accessToken: string }): Promise }; }; }>; - }; + }>(channelResponse, "YouTube channel lookup failed"); const channel = payload.items?.[0]; if (channel?.id) { @@ -937,16 +950,11 @@ async function evaluateDiscordEntitlements(options: { }; } - if (!response.ok) { - throw new ProviderHttpError(response.status, "Discord guild member lookup failed"); - } - const text = await response.text(); - - const member = (text ? JSON.parse(text) : {}) as { + const member = await readProviderJson<{ nick?: string; roles?: string[]; joined_at?: string; - }; + }>(response, "Discord guild member lookup failed"); const roles = member.roles ?? []; const matchedRoles = allowedRoleIds.filter((roleId) => roles.includes(roleId)); const facts: string[] = []; diff --git a/src/private-command-plane.ts b/src/private-command-plane.ts index bfa346b..c1912f0 100644 --- a/src/private-command-plane.ts +++ b/src/private-command-plane.ts @@ -7,7 +7,7 @@ import { type CultNetOperationServer, } from "cultnet-ts"; import { type FastifyInstance } from "fastify"; -import { getHeimdallRuntimeContext, refreshAppSession, startOAuthFlow, verifyRefreshToken } from "./app.js"; +import { errorIdentity, getHeimdallRuntimeContext, refreshAppSession, startOAuthFlow, verifyRefreshToken } from "./app.js"; import { callerFromOpenedEnvelope } from "./app-caller.js"; import { getAppProfile } from "./app-profiles.js"; import { executeHeimdallAccessPlugin, HEIMDALL_ACCESS_PLUGIN_ID, type EvePluginAbiRequest } from "./access-plugin.js"; @@ -138,6 +138,12 @@ export async function startHeimdallPrivateCommandPlane( } return response(request, cachedResult.status, cachedResult.envelope, config.daemonId); } catch (error) { + // The reply's diagnostics travel outside the sealed envelope, and the + // caller logs them. Errors caught here carry input: JSON.parse quotes + // a provider's body, a store error names the database. So the + // diagnostic is fixed text plus the error's class and code; anything + // more lives only in a sealed payload the plane authored. + const { errorClass, errorCode } = errorIdentity(error); return { schemaVersion: "cultnet.operation_response.v0", messageId: request.messageId, @@ -147,7 +153,7 @@ export async function startHeimdallPrivateCommandPlane( payloadSchema: "gamecult.cultnet.operation_failure.v1", payloadEncoding: "messagepack-base64", payload: Buffer.from(encode({ code: "private-command-denied", message: "Heimdall denied the private command." })).toString("base64"), - diagnostics: [error instanceof Error ? error.message : "Private command failed."], + diagnostics: [`Heimdall denied the private command (${errorClass}${errorCode ? `, ${errorCode}` : ""}).`], sourceRuntimeId: config.daemonId, }; } diff --git a/tests/app.test.ts b/tests/app.test.ts index a529150..374a3f4 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1542,13 +1542,29 @@ describe("the failed-callback audit event for a provider's HTTP refusal", () => expect(failed[0]!.eventPayloadJson).toEqual({ provider: "discord", mode: "sign_in", - errorClass: "ProviderHttpError", - errorCode: null, + errorClass: "ProviderResponseError", + errorCode: "provider_status", providerStatus: 401, }); expect(JSON.stringify(failed)).not.toContain("CANARY"); }); + it("keeps a 2xx body that is not JSON out of the event, by class and code alone", async () => { + const failed = await auditAfterTokenExchangeAnswers( + () => new Response('{"access_token":"CANARYaccess","refresh_token":CANARYrefresh}', { status: 200 }) + ); + + expect(failed).toHaveLength(1); + expect(failed[0]!.eventPayloadJson).toEqual({ + provider: "discord", + mode: "sign_in", + errorClass: "ProviderResponseError", + errorCode: "provider_body_not_json", + providerStatus: 200, + }); + expect(JSON.stringify(failed)).not.toContain("CANARY"); + }); + it("keeps no status that is not an HTTP status", async () => { const store = new InMemoryStore(); const app = await buildApp({ diff --git a/tests/private-command-diagnostics.test.ts b/tests/private-command-diagnostics.test.ts new file mode 100644 index 0000000..3204e10 --- /dev/null +++ b/tests/private-command-diagnostics.test.ts @@ -0,0 +1,111 @@ +// A private command's reply travels partly outside its sealed envelope: the +// status, the failure payload and the diagnostics are plaintext, and the +// calling app logs the diagnostics. Soul pass 4 found a provider's refresh +// token there: a 2xx refresh answer that is not JSON made JSON.parse quote the +// text around the fault, and the plane returned that message as a diagnostic. +// These tests drive the real Discord runtime with only fetch stubbed. +import { decode, encode } from "@msgpack/msgpack"; +import { invokeCultNetOperation } from "cultnet-ts"; +import { afterEach, expect, it, vi } from "vitest"; +import { buildApp } from "../src/app.js"; +import { type HeimdallConfig } from "../src/config.js"; +import { startHeimdallPrivateCommandPlane } from "../src/private-command-plane.js"; +import { openPrivateEnvelope, sealPrivateEnvelope, type HeimdallPrivateEnvelope } from "../src/private-command-security.js"; +import { InMemoryStore } from "../src/store/index.js"; + +const secret = "ghostlight-private-command-secret"; +const resources: Array<{ close(): Promise }> = []; +afterEach(async () => { + vi.unstubAllGlobals(); + await Promise.all(resources.splice(0).reverse().map((r) => r.close())); +}); + +function config(): HeimdallConfig { + return { + serviceName: "heimdall", host: "127.0.0.1", port: 4100, privateCommandHost: "127.0.0.1", privateCommandPort: 0, + workspaceRoot: "/tmp", dataRoot: "/tmp/.hd", cultCachePath: "/tmp/.hd/t.cc", + publicBaseUrl: "https://heimdall.gamecult.org", issuer: "https://heimdall.gamecult.org", daemonId: "yggdrasil-heimdall", + idunnRudpHealth: undefined, idunnHealthContract: "heimdall.cultnet-rudp-provider-health", providerHealthIdentityPath: "/tmp/.hd/p.cc", + sessionTtlSeconds: 3600, refreshTtlSeconds: 3600, stateTtlSeconds: 600, completionTtlSeconds: 300, + bootstrapSigningPrivateKeyOnMissing: false, tokenEncryptionKeyBase64: Buffer.alloc(32, 7).toString("base64"), + appSharedSecrets: { ghostlight: secret }, appRuntimeIds: { ghostlight: ["yggdrasil-ghostlight"] }, appBackendCallbacks: {}, + storage: { backend: "memory", applySchemaOnStartup: true }, + providers: { discord: { clientId: "discord-client", clientSecret: "discord-secret" }, patreon: {}, github: {}, twitch: {}, youtube: {}, spotify: {} }, + } as HeimdallConfig; +} + +async function command(endpoint: unknown, operation: string, contentSchema: string, key: string, payload: Record) { + const envelope = sealPrivateEnvelope({ appSlug: "ghostlight", operation, contentSchema, idempotencyKey: key, secret, payload }); + return invokeCultNetOperation(endpoint as never, { + schemaVersion: "cultnet.operation_request.v0", messageId: `m-${key}`, serviceId: "heimdall.private.commands", operation, + payloadSchema: "heimdall.private_command_envelope.v1", payloadEncoding: "messagepack-base64", + payload: Buffer.from(encode(envelope)).toString("base64"), sourceRuntimeId: "yggdrasil-ghostlight", + } as never, { runtimeId: "yggdrasil-ghostlight" }); +} +const open = (reply: { payload: string }) => + openPrivateEnvelope(decode(Buffer.from(reply.payload, "base64")) as HeimdallPrivateEnvelope, secret); + +/** A signed-in Ghostlight session, then a refresh whose provider answer is `refreshAnswer`. */ +async function refreshAgainst(refreshAnswer: () => Response) { + let refreshing = false; + vi.stubGlobal("fetch", async (url: string | URL, init?: RequestInit) => { + const u = String(url); + if (u.endsWith("/oauth2/token")) { + if (new URLSearchParams(String(init?.body)).get("grant_type") === "refresh_token") { + refreshing = true; + return refreshAnswer(); + } + return new Response( + JSON.stringify({ access_token: "first-access", refresh_token: "first-refresh", token_type: "Bearer", expires_in: 60, scope: "identify" }), + { status: 200 } + ); + } + if (u.endsWith("/users/@me")) return new Response(JSON.stringify({ id: "u1", username: "u1" }), { status: 200 }); + if (u.includes("/member")) return new Response(JSON.stringify({ roles: ["r"] }), { status: 200 }); + return new Response("{}", { status: 404 }); + }); + const cfg = config(); + const app = await buildApp({ config: cfg, store: new InMemoryStore() }); + resources.push(app); + const plane = await startHeimdallPrivateCommandPlane(app, cfg); + resources.push(plane); + const policy = { kind: "discord_role_access", guildId: "g", allowedRoleIds: ["r"] }; + const begin = open(await command(plane.endpoint, "heimdall.auth.begin", "heimdall.auth_begin_command.v1", "k1", + { provider: "discord", mode: "sign_in", returnTo: "https://yggdrasil.gamecult.org/ghostlight/", entitlementPolicy: policy })); + const state = new URL(String((begin.navigation as { url: string }).url)).searchParams.get("state")!; + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=c1&state=${encodeURIComponent(state)}`, + headers: { accept: "application/json" }, + }); + expect(callback.statusCode, callback.body).toBe(201); + const done = open(await command(plane.endpoint, "heimdall.auth.complete", "heimdall.auth_complete_command.v1", "k2", { handle: begin.handle })); + expect(done.status).toBe("authenticated"); + const reply = await command(plane.endpoint, "heimdall.auth.refresh", "heimdall.auth_refresh_command.v1", "k3", + { refreshToken: done.refreshToken, entitlementPolicy: policy }); + expect(refreshing).toBe(true); + return reply; +} + +/** Every plaintext byte of a reply: the message itself and its decoded failure payload. */ +function plaintext(reply: { payload: string }): string { + return JSON.stringify(reply) + JSON.stringify(decode(Buffer.from(reply.payload, "base64"))); +} + +it("keeps a provider's malformed 2xx refresh body out of every plaintext part of the reply", async () => { + const reply = await refreshAgainst( + () => new Response('{"access_token":"CANARYaccess0123456789","refresh_token":CANARYrefresh0123456789,"token_type":"Bearer"}', { status: 200 }) + ); + + expect(reply.status).toBe("denied"); + expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ProviderResponseError, provider_body_not_json)."]); + expect(plaintext(reply)).not.toContain("CANARY"); +}); + +it("keeps a provider's refusal body out of every plaintext part of the reply", async () => { + const reply = await refreshAgainst(() => new Response('{"error":"invalid_grant","echo":"CANARYrefused"}', { status: 400 })); + + expect(reply.status).toBe("denied"); + expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ProviderResponseError, provider_status)."]); + expect(plaintext(reply)).not.toContain("CANARY"); +}); From b5b980578c8e57105f662a0a8171a3d867e3f4fc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:07:45 +0200 Subject: [PATCH 24/85] Backend handoff failure names the status, never the app's answer Same class as Soul pass 4's F1: deliverBackendHandoff put the receiving app's whole response body into its error message. The handoff carries tokens and an app's error page can quote it, and the message reaches whatever catches the error. It now names the status alone and discards the body unread. Co-Authored-By: Claude Opus 5.5 --- src/backend-handoff.ts | 8 ++++---- tests/app.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/backend-handoff.ts b/src/backend-handoff.ts index e6cdacc..1f624e9 100644 --- a/src/backend-handoff.ts +++ b/src/backend-handoff.ts @@ -44,10 +44,10 @@ export async function deliverBackendHandoff(callbackUrl: string, payload: Backen body: JSON.stringify(payload), }); + // The status alone: the app's answer body can quote the handoff, which + // carries tokens, and whatever catches this may record or print it. if (!response.ok) { - const body = await response.text(); - throw new Error( - `Backend handoff delivery failed (status ${response.status}): ${body || "empty response"}` - ); + await response.body?.cancel(); + throw new Error(`Backend handoff delivery failed (status ${response.status}).`); } } diff --git a/tests/app.test.ts b/tests/app.test.ts index 374a3f4..2550d94 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { type FastifyInstance } from "fastify"; import { buildApp } from "../src/app.js"; +import { deliverBackendHandoff, type BackendHandoffPayload } from "../src/backend-handoff.js"; import { type HeimdallConfig, loadConfig } from "../src/config.js"; import { entitlementFacts, identityFacts } from "../src/facts.js"; import { createOAuthRuntimeRegistry, type OAuthProviderRuntime } from "../src/oauth.js"; @@ -1588,3 +1589,28 @@ describe("the failed-callback audit event for a provider's HTTP refusal", () => expect(failed?.eventPayloadJson.providerStatus).toBeNull(); }); }); + +describe("a backend handoff the app refuses", () => { + it("fails by status alone, never with the app's answer body", async () => { + globalThis.fetch = async () => new Response('{"error":"rejected","echo":"CANARYhandoff"}', { status: 500 }); + const failure = await deliverBackendHandoff("https://bifrost.gamecult.org/auth/heimdall/callback", { + source: "heimdall", + kind: "oauth_result", + handoffKind: "backend_callback", + attemptId: "attempt-1", + status: "error", + provider: "discord", + appSlug: "bifrost", + returnTo: "https://bifrost.gamecult.org/", + connection: null, + error: "oauth_callback_failed", + } as BackendHandoffPayload).then( + () => undefined, + (error: unknown) => error + ); + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toBe("Backend handoff delivery failed (status 500)."); + expect(JSON.stringify(failure) + String(failure)).not.toContain("CANARY"); + }); +}); From 2e06678301c448a2edf0cd6f9bea6c2ec1dd2b75 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:08:29 +0200 Subject: [PATCH 25/85] Patron support sync keeps Bifrost's answer by status alone Soul pass 4, F2: the route stored Bifrost's whole response body in its audit event and returned it to the caller, as `detail` on a refusal and as `bifrostResponse` on success. Another service's body can quote the signed fact or its own configuration. Both audit events now carry bifrostStatus and no body; a refusal answers 502 with fixed text and Bifrost's status; success answers bifrostStatus in place of the body, which no caller reads (Bifrost's docs name only the route). The body is discarded unread. Tests drive the route against a Bifrost that answers 200 and 403 with the canary in its body, and pin the reply and the audit event exactly. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 10 +++++---- tests/app.test.ts | 55 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/app.ts b/src/app.ts index a63117c..eaaef7d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1689,7 +1689,10 @@ export async function buildApp(options: BuildAppOptions = {}): Promise { } }); - it("syncs a linked Patreon membership into a signed Bifrost support fact", async () => { + async function syncPatronSupport(bifrostAnswer: () => Response) { const baseConfig = createTestConfig(); const config = { ...baseConfig, @@ -589,7 +589,7 @@ describe("Heimdall service", () => { bifrostBody = String(init?.body ?? ""); const headers = init?.headers as Record; bifrostSignature = headers["x-heimdall-signature-256"] ?? ""; - return new Response("processed", { status: 200 }); + return bifrostAnswer(); } return new Response("unexpected request", { status: 500 }); @@ -608,6 +608,15 @@ describe("Heimdall service", () => { }, }); + const auditEvents = [...(store as unknown as { auditEvents: Map }).auditEvents.values()]; + return { response, auditEvents, bifrostBody, bifrostSignature, now }; + } + + it("syncs a linked Patreon membership into a signed Bifrost support fact", async () => { + const { response, auditEvents, bifrostBody, bifrostSignature, now } = await syncPatronSupport( + () => new Response("processed CANARYbifrost", { status: 200 }) + ); + expect(response.statusCode, response.body).toBe(200); expect(bifrostSignature).toMatch(/^sha256=[0-9a-f]{64}$/); expect(JSON.parse(bifrostBody)).toEqual({ @@ -624,12 +633,44 @@ describe("Heimdall service", () => { providerSubscriptionId: "member-789", notes: "Verified active Patreon membership for tier Inner Sanctum.", }); - expect(response.json()).toEqual( - expect.objectContaining({ - status: "synced", - bifrostResponse: "processed", - }) + expect(response.json()).toEqual(expect.objectContaining({ status: "synced", bifrostStatus: 200 })); + expect(auditEvents.map((event) => [event.eventType, event.eventPayloadJson])).toEqual([ + [ + "bifrost_patron_support_synced", + { + provider: "patreon", + requiredTierTitle: "Inner Sanctum", + providerEventId: "patreon-membership-snapshot:patreon-user-456:member-789:2026-06-09:1500", + bifrostStatus: 200, + }, + ], + ]); + expect(response.body + JSON.stringify(auditEvents)).not.toContain("CANARY"); + }); + + it("answers a Bifrost refusal with fixed text and its status, never its body", async () => { + const { response, auditEvents } = await syncPatronSupport( + () => new Response('{"error":"bad signature","echo":"CANARYrefusal"}', { status: 403 }) ); + + expect(response.statusCode).toBe(502); + expect(response.json()).toEqual({ + error: "bifrost_patron_support_rejected", + statusCode: 403, + detail: "Bifrost refused the patron support fact.", + }); + expect(auditEvents.map((event) => [event.eventType, event.eventPayloadJson])).toEqual([ + [ + "bifrost_patron_support_sync_failed", + { + provider: "patreon", + requiredTierTitle: "Inner Sanctum", + providerEventId: "patreon-membership-snapshot:patreon-user-456:member-789:2026-06-09:1500", + bifrostStatus: 403, + }, + ], + ]); + expect(response.body + JSON.stringify(auditEvents)).not.toContain("CANARY"); }); it("redeems a completion code exactly once", async () => { From 48915317ef8cc401d3f218955ce141abae7fc7ae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:09:20 +0200 Subject: [PATCH 26/85] Recipe test observes the secrets loadConfig reads Soul pass 4, F3: the recipe test found Heimdall's secrets by parsing readSecretInput calls out of the source, and an alias, a parenthesised first argument or a colliding local name hid a new secret from it (S1-S3). The test now runs the real loadConfig against an environment that records every name looked up, standing in for process.env as well, and takes each `_FILE` lookup as a secret: readSecretInput makes one per secret, however the call is spelled and whatever list computes the name. Each must be declared as `_FILE` in the recipe and never under its plaintext name. The source scan and its table of computed names are deleted. Co-Authored-By: Claude Opus 5.5 --- tests/recipe-secrets.test.ts | 67 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/tests/recipe-secrets.test.ts b/tests/recipe-secrets.test.ts index f184e63..ec477a0 100644 --- a/tests/recipe-secrets.test.ts +++ b/tests/recipe-secrets.test.ts @@ -1,7 +1,8 @@ -import { readdirSync, readFileSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { appProfiles } from "../src/app-profiles.js"; import { appSlugs, providers } from "../src/contracts.js"; +import { loadConfig } from "../src/config.js"; import { providerCatalog } from "../src/providers.js"; // The Idunn recipe decides which environment names a binding may set, and @@ -26,34 +27,40 @@ const everyDeclaredName = [...recipe.replace(/\r\n/g, "\n").matchAll(/^\w+_envir (match) => [...match[1]!.matchAll(/"([^"]+)"/g)].map((name) => name[1]!) ); -// The secrets Heimdall reads are whatever its source passes to -// readSecretInput, so the set is read from the source: a secret added there -// cannot be declared in plaintext here without failing. A literal name is -// taken as written; the two computed names expand over the lists that compute -// them; any other argument fails, because this test cannot know its names. -const srcRoot = new URL("../src/", import.meta.url); -const secretArguments = readdirSync(srcRoot, { recursive: true }) - .map(String) - .filter((file) => file.endsWith(".ts")) - .flatMap((file) => - [...readFileSync(new URL(file.replace(/\\/g, "/"), srcRoot), "utf8").matchAll(/(? call[1]! - ) - ); - -const computedSecretNames: Record = { - "`${prefix}_CLIENT_SECRET`": providers.map((provider) => `GC_ACCESS_PROVIDER_${provider.toUpperCase()}_CLIENT_SECRET`), - envKey: appSlugs.map((appSlug) => `GC_ACCESS_APP_${appSlug.toUpperCase()}_SHARED_SECRET`), -}; +// The secrets Heimdall reads are observed, not read from the source: the real +// loadConfig runs against an environment that records every name it looks +// up. readSecretInput looks up `NAME_FILE` for each secret, so every `_FILE` +// lookup names one, however the call is spelled and whatever list computes +// the name. The same recorder stands in for process.env during the call, so a +// read that bypasses the env argument is seen too. The environment is empty, +// so this sees the reads loadConfig makes when nothing is set; a secret read +// only on a branch some other variable opens would not be seen. +function namesLoadConfigReads(): Set { + const names = new Set(); + const env = new Proxy({} as NodeJS.ProcessEnv, { + get(_target, name) { + if (typeof name === "string") names.add(name); + return undefined; + }, + has(_target, name) { + if (typeof name === "string") names.add(name); + return false; + }, + }); + const processEnv = process.env; + process.env = env; + try { + loadConfig(env, []); + } finally { + process.env = processEnv; + } + return names; +} /** Secrets Heimdall reads through readSecretInput, by their plaintext name. */ -const secretNames = secretArguments.flatMap((argument) => { - const literal = /^"([A-Z0-9_]+)"$/.exec(argument); - if (literal) return [literal[1]!]; - const computed = computedSecretNames[argument]; - if (computed) return computed; - throw new Error(`recipe-secrets cannot derive the names read by readSecretInput(env, ${argument}); teach it.`); -}); +const secretNames = [...namesLoadConfigReads()] + .filter((name) => name.endsWith("_FILE")) + .map((name) => name.slice(0, -"_FILE".length)); const secretShaped = /SECRET|PASSWORD|PASSPHRASE|TOKEN|API_KEY|PRIVATE_KEY|ENCRYPTION_KEY|SALT|CREDENTIAL|DATABASE_URL|_DSN|_PEM/; @@ -79,14 +86,14 @@ describe("Idunn recipe secret declarations", () => { expect(plaintext).toEqual([]); }); - it("finds every readSecretInput call in the source", () => { + it("sees every secret loadConfig reads, across the provider and app lists", () => { expect(secretNames).toEqual( expect.arrayContaining([ "GC_ACCESS_DATABASE_URL", "GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64", "GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET", - "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET", - "GC_ACCESS_APP_GHOSTLIGHT_SHARED_SECRET", + ...providers.map((provider) => `GC_ACCESS_PROVIDER_${provider.toUpperCase()}_CLIENT_SECRET`), + ...appSlugs.map((appSlug) => `GC_ACCESS_APP_${appSlug.toUpperCase()}_SHARED_SECRET`), ]) ); }); From 8541ae3f2f9ee2866851758577dc83a67d72e811 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:15:34 +0200 Subject: [PATCH 27/85] Startup proves the database can take Heimdall's writes Soul pass 4, F4: the startup check passed on databases where every request still failed: a role granted SELECT only (O2), and a table whose primary key and key column were dropped (N), including on a start that applied the schema, since CREATE ... IF NOT EXISTS leaves a damaged table as it is. Every start now ends in checkSchema, after applying the schema or not. It requires, read from the schema SQL: every relation; every column the created tables declare, not only the added one (read from pg_attribute, so a column the role may not see is not reported as missing); every primary key and unique constraint as a non-partial unique index on exactly its columns, which is what the store's ON CONFLICT clauses and the foreign keys rely on; and SELECT, INSERT and UPDATE on every table. A missing piece refuses with SCHEMA_MISSING, a missing privilege with SCHEMA_PRIVILEGES, through the same fixed startup message. A live-Postgres block drives createStore itself, skipped unless HEIMDALL_TEST_PG_ADMIN_URL names a role that may create databases and roles: a read-only role (and its canary password stays out of the message), a dropped primary key with and without applying, a unique key replaced by a partial one, and a dropped column. Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 11 ++- src/store/postgres.ts | 65 +++++++++++++---- src/store/schema.ts | 59 +++++++++++++-- tests/store.test.ts | 166 +++++++++++++++++++++++++++++++++++++----- 4 files changed, 261 insertions(+), 40 deletions(-) diff --git a/src/store/index.ts b/src/store/index.ts index b281ec4..32f6e8e 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -14,10 +14,15 @@ export async function createStore(config: HeimdallConfig): Promise { - const result = await this.pool.query<{ missing: string }>( + const result = await this.pool.query<{ kind: "missing" | "privilege" }>( ` - SELECT name AS missing FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL + SELECT 'missing' AS kind FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL UNION ALL - SELECT required.table_name || '.' || required.column_name - FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name) + SELECT 'missing' FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name) WHERE NOT EXISTS ( - SELECT 1 FROM information_schema.columns c - WHERE c.table_schema = current_schema() - AND c.table_name = required.table_name - AND c.column_name = required.column_name + SELECT 1 FROM pg_attribute a + WHERE a.attrelid = to_regclass(required.table_name) + AND a.attname = required.column_name + AND a.attnum > 0 + AND NOT a.attisdropped ) + UNION ALL + SELECT 'missing' FROM unnest($4::text[], $5::text[], $6::boolean[]) AS required(table_name, columns, is_primary) + WHERE NOT EXISTS ( + SELECT 1 FROM pg_index i + WHERE i.indrelid = to_regclass(required.table_name) + AND i.indisunique + AND i.indpred IS NULL + AND (i.indisprimary OR NOT required.is_primary) + AND i.indnatts = cardinality(string_to_array(required.columns, ',')) + AND ( + SELECT string_agg(a.attname, ',' ORDER BY a.attname) + FROM unnest(i.indkey::int2[]) AS key(attnum) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum + ) = required.columns + ) + UNION ALL + SELECT 'privilege' FROM unnest($7::text[]) AS required(table_name) + CROSS JOIN unnest($8::text[]) AS wanted(privilege) + WHERE to_regclass(required.table_name) IS NOT NULL + AND NOT has_table_privilege(to_regclass(required.table_name), wanted.privilege) `, [ REQUIRED_RELATIONS, REQUIRED_COLUMNS.map((required) => required.table), REQUIRED_COLUMNS.map((required) => required.column), + REQUIRED_KEYS.map((required) => required.table), + REQUIRED_KEYS.map((required) => required.columns), + REQUIRED_KEYS.map((required) => required.primary), + REQUIRED_TABLES, + REQUIRED_TABLE_PRIVILEGES, ] ); - if (result.rows.length > 0) { + if (result.rows.some((row) => row.kind === "missing")) { throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); } + if (result.rows.length > 0) { + throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { + code: "SCHEMA_PRIVILEGES", + }); + } } async close(): Promise { diff --git a/src/store/schema.ts b/src/store/schema.ts index 8e9c921..cb80eb0 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -163,15 +163,60 @@ CREATE INDEX IF NOT EXISTS audit_events_lookup_idx ON audit_events(account_id, session_id, app_slug, created_at); `; -/** - * What a start that does not apply the schema must find, read from the SQL - * above so the two cannot drift: every table and index it creates, and every - * column it adds to an existing table. +const schemaText = CREATE_SCHEMA_SQL.replace(/\r\n/g, "\n"); +const createdTables = [...schemaText.matchAll(/^CREATE TABLE IF NOT EXISTS (\w+) \(\n([\s\S]*?)\n\);/gm)].map((match) => ({ + table: match[1]!, + lines: match[2]!.split("\n").map((line) => line.trim().replace(/,$/, "")), +})); + +/** The sorted, comma-separated columns of a `PRIMARY KEY (a, b)` or `UNIQUE(a, b)` line. */ +function keyColumns(line: string, keyword: "PRIMARY KEY" | "UNIQUE"): string | undefined { + const match = new RegExp(`^${keyword}\\s*\\(([^)]*)\\)`).exec(line); + return match ? match[1]!.split(",").map((column) => column.trim()).sort().join(",") : undefined; +} + +/* + * What every start must find before it serves, read from the SQL above so the + * two cannot drift. A start that applies the schema checks as well, because + * CREATE ... IF NOT EXISTS leaves a damaged table as it is. */ + +/** Every table and index the schema creates. */ export const REQUIRED_RELATIONS = [ - ...CREATE_SCHEMA_SQL.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), + ...schemaText.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), ].map((match) => match[1]!); +/** Every table the schema creates. The store reads and writes each one. */ +export const REQUIRED_TABLES = createdTables.map(({ table }) => table); + +/** What the store does to its tables: it reads, inserts and updates, and never deletes. */ +export const REQUIRED_TABLE_PRIVILEGES = ["SELECT", "INSERT", "UPDATE"]; + +/** Every column a created table declares, and every column the schema adds to an existing table. */ export const REQUIRED_COLUMNS = [ - ...CREATE_SCHEMA_SQL.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm), -].map((match) => ({ table: match[1]!, column: match[2]! })); + ...createdTables.flatMap(({ table, lines }) => + lines + .filter((line) => !/^(?:PRIMARY KEY|UNIQUE)\b/.test(line)) + .map((line) => ({ table, column: /^(\w+)/.exec(line)![1]! })) + ), + ...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm)].map((match) => ({ + table: match[1]!, + column: match[2]!, + })), +]; + +/** + * Every primary key and unique constraint a created table declares, by its + * sorted columns. Foreign keys reference the primary keys, and each of the + * store's ON CONFLICT clauses needs a unique index on exactly its columns: + * without one, every such write fails. + */ +export const REQUIRED_KEYS = createdTables.flatMap(({ table, lines }) => + lines.flatMap((line) => { + const inline = /^(\w+) .*\bPRIMARY KEY\b/.exec(line); + const columns = inline?.[1] ?? keyColumns(line, "PRIMARY KEY"); + if (columns) return [{ table, columns, primary: true }]; + const unique = keyColumns(line, "UNIQUE"); + return unique ? [{ table, columns: unique, primary: false }] : []; + }) +); diff --git a/tests/store.test.ts b/tests/store.test.ts index c28e353..5f9f5a5 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -1,7 +1,17 @@ -import { describe, expect, it } from "vitest"; +import { randomUUID } from "node:crypto"; +import { Pool } from "pg"; +import { afterEach, describe, expect, it } from "vitest"; import { newDb } from "pg-mem"; +import { type HeimdallConfig } from "../src/config.js"; +import { createStore } from "../src/store/index.js"; import { createPostgresStore, PostgresStore } from "../src/store/postgres.js"; -import { REQUIRED_COLUMNS, REQUIRED_RELATIONS } from "../src/store/schema.js"; +import { + REQUIRED_COLUMNS, + REQUIRED_KEYS, + REQUIRED_RELATIONS, + REQUIRED_TABLE_PRIVILEGES, + REQUIRED_TABLES, +} from "../src/store/schema.js"; describe("PostgresStore", () => { it("round-trips core auth records through postgres", async () => { @@ -212,33 +222,155 @@ describe("createPostgresStore", () => { }); describe("PostgresStore.checkSchema", () => { - it("requires every table and index the schema creates and every column it adds", () => { + it("requires every table, index, column and key the schema makes", () => { expect(REQUIRED_RELATIONS).toEqual( expect.arrayContaining(["accounts", "audit_events", "auth_completions_attempt_unconsumed_unique_idx", "audit_events_lookup_idx"]) ); expect(REQUIRED_RELATIONS).toHaveLength(17); - expect(REQUIRED_COLUMNS).toEqual([{ table: "auth_completions", column: "attempt_id" }]); + expect(REQUIRED_TABLES).toEqual([ + "accounts", + "linked_identities", + "sessions", + "auth_attempts", + "private_command_receipts", + "auth_completions", + "capability_grants", + "entitlement_snapshots", + "audit_events", + ]); + expect(REQUIRED_COLUMNS).toHaveLength(85); + expect(REQUIRED_COLUMNS).toEqual( + expect.arrayContaining([ + { table: "auth_completions", column: "code" }, + { table: "auth_completions", column: "mode" }, + { table: "linked_identities", column: "profile_json" }, + { table: "auth_completions", column: "attempt_id" }, + ]) + ); + // The keys the store's ON CONFLICT clauses and the foreign keys rely on. + expect(REQUIRED_KEYS).toEqual( + expect.arrayContaining([ + { table: "linked_identities", columns: "provider,provider_user_id", primary: false }, + { table: "sessions", columns: "id", primary: true }, + { table: "private_command_receipts", columns: "app_slug,idempotency_key", primary: true }, + { table: "entitlement_snapshots", columns: "account_id,provider,scope", primary: false }, + { table: "accounts", columns: "id", primary: true }, + { table: "auth_completions", columns: "code", primary: true }, + ]) + ); + expect(REQUIRED_KEYS).toHaveLength(11); + expect(REQUIRED_TABLE_PRIVILEGES).toEqual(["SELECT", "INSERT", "UPDATE"]); }); - function storeAnswering(missing: string[]) { - const queries: unknown[][] = []; - const store = new PostgresStore({ - query: (async (_text: string, values: unknown[]) => { - queries.push(values); - return { rows: missing.map((name) => ({ missing: name })) }; - }) as never, + function storeAnswering(kinds: string[]) { + return new PostgresStore({ + query: (async () => ({ rows: kinds.map((kind) => ({ kind })) })) as never, end: async () => undefined, }); - return { store, queries }; } - it("refuses with SCHEMA_MISSING when anything is missing", async () => { - const { store, queries } = storeAnswering(["auth_completions.attempt_id"]); - await expect(store.checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" }); - expect(queries).toEqual([[REQUIRED_RELATIONS, ["auth_completions"], ["attempt_id"]]]); + it("refuses with SCHEMA_MISSING when anything is missing, whatever else is", async () => { + await expect(storeAnswering(["privilege", "missing"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" }); + }); + + it("refuses with SCHEMA_PRIVILEGES when only a privilege is missing", async () => { + await expect(storeAnswering(["privilege"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_PRIVILEGES" }); }); it("passes when nothing is missing", async () => { - await expect(storeAnswering([]).store.checkSchema()).resolves.toBeUndefined(); + await expect(storeAnswering([]).checkSchema()).resolves.toBeUndefined(); + }); +}); + +// Against a real Postgres, through the startup path itself: createStore, with +// and without applying the schema. HEIMDALL_TEST_PG_ADMIN_URL names a role +// that may create databases and roles; each case gets its own database. +const adminUrl = process.env.HEIMDALL_TEST_PG_ADMIN_URL; +describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_ADMIN_URL)", () => { + const refusal = (code: string) => `Postgres storage could not be prepared (${code}); check GC_ACCESS_DATABASE_URL_FILE.`; + const cleanup: Array<() => Promise> = []; + afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); + }); + + function urlFor(database: string, user?: string, password?: string): string { + const url = new URL(adminUrl!); + url.pathname = `/${database}`; + if (user) url.username = user; + if (password) url.password = password; + return url.toString(); + } + + /** A fresh database with the schema applied by its owner, and an owner pool on it. */ + async function appliedDatabase(): Promise<{ name: string; pool: Pool }> { + const name = `heimdall_check_${randomUUID().replace(/-/g, "")}`; + const admin = new Pool({ connectionString: adminUrl }); + await admin.query(`CREATE DATABASE ${name}`); + cleanup.push(async () => { + await admin.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); + await admin.end(); + }); + const pool = new Pool({ connectionString: urlFor(name) }); + cleanup.push(async () => { + await pool.end(); + }); + await new PostgresStore(pool).ensureSchema(); + return { name, pool }; + } + + async function start(url: string, applySchemaOnStartup: boolean): Promise { + try { + const store = await createStore({ storage: { backend: "postgres", databaseUrl: url, applySchemaOnStartup } } as HeimdallConfig); + await store.close(); + return "started"; + } catch (error) { + return (error as Error).message; + } + } + + it("starts on an applied database, applying or not", async () => { + const { name } = await appliedDatabase(); + expect(await start(urlFor(name), false)).toBe("started"); + expect(await start(urlFor(name), true)).toBe("started"); + }); + + it("refuses a role that may only read, by code and without its password", async () => { + const { name, pool } = await appliedDatabase(); + const role = `heimdall_ro_${randomUUID().replace(/-/g, "").slice(0, 12)}`; + await pool.query(`CREATE ROLE ${role} LOGIN PASSWORD 'CANARYrolepw'`); + cleanup.push(async () => { + await pool.query(`REVOKE ALL ON ALL TABLES IN SCHEMA public FROM ${role}`); + await pool.query(`DROP ROLE IF EXISTS ${role}`); + }); + await pool.query(`GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${role}`); + + const message = await start(urlFor(name, role, "CANARYrolepw"), false); + expect(message).toBe(refusal("SCHEMA_PRIVILEGES")); + expect(message).not.toContain("CANARY"); + }); + + it("refuses a table whose primary key was dropped, even after applying the schema", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE auth_completions DROP CONSTRAINT auth_completions_pkey"); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + expect(await start(urlFor(name), true)).toBe(refusal("SCHEMA_MISSING")); + }); + + it("refuses a unique key the store's ON CONFLICT needs, replaced by a partial one", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key"); + await pool.query( + "CREATE UNIQUE INDEX entitlement_snapshots_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed" + ); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + }); + + it("refuses a created table's dropped column", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE auth_completions DROP COLUMN mode"); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); }); }); From 7be541467dcab2bb4299f568498ff117296aba76 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:16:21 +0200 Subject: [PATCH 28/85] Schema check: a key is a unique index on its columns, primary or not The store's ON CONFLICT clauses and the foreign keys need a unique, non-partial index on exactly the key's columns; whether it is the primary key changes nothing a request does. Requiring the primary flag added a refusal no failing request motivated, so it is dropped. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 8 +++----- src/store/schema.ts | 14 ++++++-------- tests/store.test.ts | 12 ++++++------ 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 8051446..ab0765a 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -319,13 +319,12 @@ export class PostgresStore implements HeimdallStore { AND NOT a.attisdropped ) UNION ALL - SELECT 'missing' FROM unnest($4::text[], $5::text[], $6::boolean[]) AS required(table_name, columns, is_primary) + SELECT 'missing' FROM unnest($4::text[], $5::text[]) AS required(table_name, columns) WHERE NOT EXISTS ( SELECT 1 FROM pg_index i WHERE i.indrelid = to_regclass(required.table_name) AND i.indisunique AND i.indpred IS NULL - AND (i.indisprimary OR NOT required.is_primary) AND i.indnatts = cardinality(string_to_array(required.columns, ',')) AND ( SELECT string_agg(a.attname, ',' ORDER BY a.attname) @@ -334,8 +333,8 @@ export class PostgresStore implements HeimdallStore { ) = required.columns ) UNION ALL - SELECT 'privilege' FROM unnest($7::text[]) AS required(table_name) - CROSS JOIN unnest($8::text[]) AS wanted(privilege) + SELECT 'privilege' FROM unnest($6::text[]) AS required(table_name) + CROSS JOIN unnest($7::text[]) AS wanted(privilege) WHERE to_regclass(required.table_name) IS NOT NULL AND NOT has_table_privilege(to_regclass(required.table_name), wanted.privilege) `, @@ -345,7 +344,6 @@ export class PostgresStore implements HeimdallStore { REQUIRED_COLUMNS.map((required) => required.column), REQUIRED_KEYS.map((required) => required.table), REQUIRED_KEYS.map((required) => required.columns), - REQUIRED_KEYS.map((required) => required.primary), REQUIRED_TABLES, REQUIRED_TABLE_PRIVILEGES, ] diff --git a/src/store/schema.ts b/src/store/schema.ts index cb80eb0..de39316 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -207,16 +207,14 @@ export const REQUIRED_COLUMNS = [ /** * Every primary key and unique constraint a created table declares, by its - * sorted columns. Foreign keys reference the primary keys, and each of the - * store's ON CONFLICT clauses needs a unique index on exactly its columns: - * without one, every such write fails. + * sorted columns. Each must stand as a unique, non-partial index on exactly + * those columns: the store's ON CONFLICT clauses and the foreign keys need + * nothing more, and without it every such write fails. */ export const REQUIRED_KEYS = createdTables.flatMap(({ table, lines }) => lines.flatMap((line) => { - const inline = /^(\w+) .*\bPRIMARY KEY\b/.exec(line); - const columns = inline?.[1] ?? keyColumns(line, "PRIMARY KEY"); - if (columns) return [{ table, columns, primary: true }]; - const unique = keyColumns(line, "UNIQUE"); - return unique ? [{ table, columns: unique, primary: false }] : []; + const columns = + /^(\w+) .*\bPRIMARY KEY\b/.exec(line)?.[1] ?? keyColumns(line, "PRIMARY KEY") ?? keyColumns(line, "UNIQUE"); + return columns ? [{ table, columns }] : []; }) ); diff --git a/tests/store.test.ts b/tests/store.test.ts index 5f9f5a5..0729ba0 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -250,12 +250,12 @@ describe("PostgresStore.checkSchema", () => { // The keys the store's ON CONFLICT clauses and the foreign keys rely on. expect(REQUIRED_KEYS).toEqual( expect.arrayContaining([ - { table: "linked_identities", columns: "provider,provider_user_id", primary: false }, - { table: "sessions", columns: "id", primary: true }, - { table: "private_command_receipts", columns: "app_slug,idempotency_key", primary: true }, - { table: "entitlement_snapshots", columns: "account_id,provider,scope", primary: false }, - { table: "accounts", columns: "id", primary: true }, - { table: "auth_completions", columns: "code", primary: true }, + { table: "linked_identities", columns: "provider,provider_user_id" }, + { table: "sessions", columns: "id" }, + { table: "private_command_receipts", columns: "app_slug,idempotency_key" }, + { table: "entitlement_snapshots", columns: "account_id,provider,scope" }, + { table: "accounts", columns: "id" }, + { table: "auth_completions", columns: "code" }, ]) ); expect(REQUIRED_KEYS).toHaveLength(11); From 804f41f5f7ae291cecc2f5aa58354456eb10205a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:16:39 +0200 Subject: [PATCH 29/85] Pin that no provider error carries the provider's body F1's second half, tested where it is decided: the Discord token exchange (a 2xx body that is not JSON, and a 401), the Discord guild lookup and the YouTube channel lookup each fail with a ProviderResponseError whose code and status name the failure, and no printed form of the error (message, stack, JSON) holds a byte of the canary body. Until now only the callers' fixed answers were pinned, so a body in the message went unseen. Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/app.test.ts b/tests/app.test.ts index c03caf5..b3ae9d3 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1655,3 +1655,67 @@ describe("a backend handoff the app refuses", () => { expect(JSON.stringify(failure) + String(failure)).not.toContain("CANARY"); }); }); + +describe("a provider answer Heimdall cannot use", () => { + const runtimes = createOAuthRuntimeRegistry(); + const notJson = '{"access_token":"CANARYaccess","refresh_token":CANARYrefresh}'; + const refusal = '{"error":"invalid_grant","echo":"CANARYrefused"}'; + + /** What the call threw, as every form a catcher might record or print. */ + async function failureOf(call: () => Promise) { + const error = await call().then( + () => undefined, + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + return { + error: error as Error & { code?: unknown; status?: unknown }, + printed: `${String(error)} ${(error as Error).stack ?? ""} ${JSON.stringify(error)}`, + }; + } + + it.each([ + ["a 2xx body that is not JSON", () => new Response(notJson, { status: 200 }), "provider_body_not_json", 200], + ["a refusal", () => new Response(refusal, { status: 401 }), "provider_status", 401], + ])("fails the Discord token exchange on %s by status and code alone", async (_name, answer, code, status) => { + globalThis.fetch = async () => answer(); + const { error, printed } = await failureOf(() => + runtimes.discord.exchangeAuthorizationCode({ config: createTestConfig(), code: "c", redirectUri: "https://heimdall.gamecult.org/cb" }) + ); + + expect(error.name).toBe("ProviderResponseError"); + expect(error.code).toBe(code); + expect(error.status).toBe(status); + expect(printed).not.toContain("CANARY"); + }); + + it("fails the Discord guild lookup on a 2xx body that is not JSON by status and code alone", async () => { + globalThis.fetch = async () => new Response(notJson, { status: 200 }); + const { error, printed } = await failureOf(() => + runtimes.discord.evaluateEntitlements({ + config: createTestConfig(), + callback: { + appSlug: "ghostlight", + accountId: "account-1", + connection: null, + entitlementPolicy: { kind: "discord_role_access", guildId: "g", allowedRoleIds: ["r"] }, + }, + identity: { provider: "discord", providerUserId: "u1", profile: {} }, + tokenSet: { accessToken: "a", tokenType: "Bearer", scope: [], raw: {} }, + }) + ); + + expect(error.code).toBe("provider_body_not_json"); + expect(printed).not.toContain("CANARY"); + }); + + it("fails the YouTube channel lookup on a 2xx body that is not JSON by status and code alone", async () => { + globalThis.fetch = async () => new Response(notJson, { status: 200 }); + const { error, printed } = await failureOf(() => + runtimes.youtube.resolveIdentity({ config: createTestConfig(), accessToken: "a" }) + ); + + expect(error.code).toBe("provider_body_not_json"); + expect(printed).not.toContain("CANARY"); + }); +}); From b5d7b6b0cceb474496cecd9fadc80f7a790db11b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:29:19 +0200 Subject: [PATCH 30/85] A refused store closes its pool; pin the key's columns and uniqueness The startup check's refusal left the store's pool open: the process exits on it in production, but the connection outlived the refusal. createStore now closes a store it refuses. Two live cases the first run showed undefended (mutants D6 and D9): a unique key replaced by a plain index on the same columns, and by a unique index on other columns of the same count. The read-only-role case also checks that no session of the role outlives the refusal. The fixture's owner pool ignores the connection ends its own forced drop causes. Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 3 +++ tests/store.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/store/index.ts b/src/store/index.ts index 32f6e8e..972bd8f 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -29,6 +29,9 @@ export async function createStore(config: HeimdallConfig): Promise undefined); throw new Error( `Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); check GC_ACCESS_DATABASE_URL_FILE.` ); diff --git a/tests/store.test.ts b/tests/store.test.ts index 0729ba0..b208956 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -311,6 +311,9 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD await admin.end(); }); const pool = new Pool({ connectionString: urlFor(name) }); + // The forced drop ends this fixture pool's connections, possibly while + // they are still closing; that is cleanup, not a finding. + pool.on("error", () => undefined); cleanup.push(async () => { await pool.end(); }); @@ -347,6 +350,15 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD const message = await start(urlFor(name, role, "CANARYrolepw"), false); expect(message).toBe(refusal("SCHEMA_PRIVILEGES")); expect(message).not.toContain("CANARY"); + + // The refused store closed its pool: no session of the role outlives the refusal. + let sessions = -1; + for (let attempt = 0; attempt < 20 && sessions !== 0; attempt += 1) { + if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, 100)); + const result = await pool.query<{ n: number }>("SELECT count(*)::int AS n FROM pg_stat_activity WHERE usename = $1", [role]); + sessions = result.rows[0]!.n; + } + expect(sessions).toBe(0); }); it("refuses a table whose primary key was dropped, even after applying the schema", async () => { @@ -367,6 +379,22 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); }); + it("refuses a unique key replaced by a plain index on the same columns", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key"); + await pool.query("CREATE INDEX linked_identities_plain ON linked_identities(provider, provider_user_id)"); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + }); + + it("refuses a unique key replaced by one on other columns of the same count", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key"); + await pool.query("CREATE UNIQUE INDEX linked_identities_other ON linked_identities(provider, username)"); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + }); + it("refuses a created table's dropped column", async () => { const { name, pool } = await appliedDatabase(); await pool.query("ALTER TABLE auth_completions DROP COLUMN mode"); From 731ce08cfaf0d3d82c4a437e108019855b27921f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:55:45 +0200 Subject: [PATCH 31/85] Schema check: keys as the store relies on them, typed columns, per-table privileges Soul pass 5, S5-1 and S5-2. checkSchema passed four databases the store could not serve, and demanded privileges the store never uses. Keys: every primary key and unique constraint, and each unique index the schema creates by name, must stand as an immediate (not deferrable) unique index whose key columns, counted by indnkeyatts and excluding INCLUDE columns, are exactly the declared ones, with exactly the declared predicate. A named index is checked by its name, so the partial attempt index R21.1 depends on cannot be swapped for a plain or unpredicated one of the same name. Predicates compare with case, parentheses and white space removed, the form both the schema text and pg_get_expr reduce to. Columns: each carries its declared type, compared as a type OID through to_regtype. Privileges: STORE_TABLE_PRIVILEGES in postgres.ts says what the store's statements do to each table. audit_events needs INSERT only; capability_grants and private_command_receipts need no UPDATE; nothing needs DELETE. REQUIRED_TABLE_PRIVILEGES and REQUIRED_TABLES are gone. The live block now proves the map both ways against a real Postgres: a role granted exactly the map starts and runs every store operation (the test checks it called every method), and for each privilege in the map, a role granted all but that one is refused at startup and fails the same workload with 42501. Each damage Soul found (a DEFERRABLE key, an INCLUDE key, the attempt index replaced by a plain one of the same name, a retyped column), plus an expression-widened key, an unpredicated or re-predicated attempt index, is refused; a key index with extra INCLUDE columns starts. Every refusal also proves its store left no connection open (tagged by application_name), on the no-apply, apply and 42501 paths. Cases clone a template database and the block has a 120 s budget, instead of building the schema per case under the 5 s default that timed out. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 61 +++++--- src/store/schema.ts | 72 ++++++--- tests/store.test.ts | 341 +++++++++++++++++++++++++++++------------- 3 files changed, 330 insertions(+), 144 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index ab0765a..ea47ca0 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -1,14 +1,7 @@ import { randomUUID } from "node:crypto"; import { Pool, type PoolClient, type QueryResultRow } from "pg"; import { type AppSlug, type HeimdallAuthAttemptStatus, type LinkedIdentityInput, type Provider } from "../contracts.js"; -import { - CREATE_SCHEMA_SQL, - REQUIRED_COLUMNS, - REQUIRED_KEYS, - REQUIRED_RELATIONS, - REQUIRED_TABLE_PRIVILEGES, - REQUIRED_TABLES, -} from "./schema.js"; +import { CREATE_SCHEMA_SQL, REQUIRED_COLUMNS, REQUIRED_KEYS, REQUIRED_RELATIONS } from "./schema.js"; import { type CreateAccountInput, type CreateAuthAttemptInput, @@ -291,6 +284,24 @@ function mapAuthAttemptRow(row: AuthAttemptRow): StoredAuthAttempt { return attempt; } +/** + * What the store's statements do to each table, and so the privileges its + * database role needs: it reads what it returns or looks up, inserts what it + * creates, updates what it changes or upserts, and deletes nothing. + * audit_events is written and never read back. + */ +export const STORE_TABLE_PRIVILEGES: Readonly> = { + accounts: ["SELECT", "INSERT", "UPDATE"], + linked_identities: ["SELECT", "INSERT", "UPDATE"], + sessions: ["SELECT", "INSERT", "UPDATE"], + auth_attempts: ["SELECT", "INSERT", "UPDATE"], + private_command_receipts: ["SELECT", "INSERT"], + auth_completions: ["SELECT", "INSERT", "UPDATE"], + capability_grants: ["SELECT", "INSERT"], + entitlement_snapshots: ["SELECT", "INSERT", "UPDATE"], + audit_events: ["INSERT"], +}; + export class PostgresStore implements HeimdallStore { constructor(private readonly pool: Pick) {} @@ -300,52 +311,62 @@ export class PostgresStore implements HeimdallStore { /** * Proves, before Heimdall serves, that the database can take its requests: - * every relation, column and key the schema makes is there, and the role - * may read, insert and update every table. A missing piece fails with code + * every relation the schema makes, every column with its declared type, + * every unique key as the store relies on it, and every privilege in + * STORE_TABLE_PRIVILEGES. A missing or altered piece fails with code * SCHEMA_MISSING and a missing privilege with SCHEMA_PRIVILEGES; neither * error names the piece. */ async checkSchema(): Promise { + const privileges = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => + wanted.map((privilege) => ({ table, privilege })) + ); const result = await this.pool.query<{ kind: "missing" | "privilege" }>( ` SELECT 'missing' AS kind FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL UNION ALL - SELECT 'missing' FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name) + SELECT 'missing' FROM unnest($2::text[], $3::text[], $4::text[]) AS required(table_name, column_name, type_name) WHERE NOT EXISTS ( SELECT 1 FROM pg_attribute a WHERE a.attrelid = to_regclass(required.table_name) AND a.attname = required.column_name AND a.attnum > 0 AND NOT a.attisdropped + AND a.atttypid = to_regtype(required.type_name) ) UNION ALL - SELECT 'missing' FROM unnest($4::text[], $5::text[]) AS required(table_name, columns) + SELECT 'missing' FROM unnest($5::text[], $6::text[], $7::text[], $8::text[]) AS required(table_name, columns, index_name, predicate) WHERE NOT EXISTS ( SELECT 1 FROM pg_index i WHERE i.indrelid = to_regclass(required.table_name) + AND (required.index_name = '' OR i.indexrelid = to_regclass(required.index_name)) AND i.indisunique - AND i.indpred IS NULL - AND i.indnatts = cardinality(string_to_array(required.columns, ',')) + AND i.indimmediate + AND coalesce(regexp_replace(lower(pg_get_expr(i.indpred, i.indrelid)), '[()[:space:]]', '', 'g'), '') = required.predicate + AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) AND ( SELECT string_agg(a.attname, ',' ORDER BY a.attname) - FROM unnest(i.indkey::int2[]) AS key(attnum) + FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum + WHERE key.position <= i.indnkeyatts ) = required.columns ) UNION ALL - SELECT 'privilege' FROM unnest($6::text[]) AS required(table_name) - CROSS JOIN unnest($7::text[]) AS wanted(privilege) + SELECT 'privilege' FROM unnest($9::text[], $10::text[]) AS required(table_name, privilege) WHERE to_regclass(required.table_name) IS NOT NULL - AND NOT has_table_privilege(to_regclass(required.table_name), wanted.privilege) + AND NOT has_table_privilege(to_regclass(required.table_name), required.privilege) `, [ REQUIRED_RELATIONS, REQUIRED_COLUMNS.map((required) => required.table), REQUIRED_COLUMNS.map((required) => required.column), + REQUIRED_COLUMNS.map((required) => required.type), REQUIRED_KEYS.map((required) => required.table), REQUIRED_KEYS.map((required) => required.columns), - REQUIRED_TABLES, - REQUIRED_TABLE_PRIVILEGES, + REQUIRED_KEYS.map((required) => required.index), + REQUIRED_KEYS.map((required) => required.predicate), + privileges.map((required) => required.table), + privileges.map((required) => required.privilege), ] ); if (result.rows.some((row) => row.kind === "missing")) { diff --git a/src/store/schema.ts b/src/store/schema.ts index de39316..f385031 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -169,10 +169,24 @@ const createdTables = [...schemaText.matchAll(/^CREATE TABLE IF NOT EXISTS (\w+) lines: match[2]!.split("\n").map((line) => line.trim().replace(/,$/, "")), })); -/** The sorted, comma-separated columns of a `PRIMARY KEY (a, b)` or `UNIQUE(a, b)` line. */ +/** Sorted and comma-separated, the form the schema check compares key columns in. */ +function columnList(text: string): string { + return text.split(",").map((column) => column.trim()).sort().join(","); +} + +/** The columns of a `PRIMARY KEY (a, b)` or `UNIQUE(a, b)` line, as a column list. */ function keyColumns(line: string, keyword: "PRIMARY KEY" | "UNIQUE"): string | undefined { const match = new RegExp(`^${keyword}\\s*\\(([^)]*)\\)`).exec(line); - return match ? match[1]!.split(",").map((column) => column.trim()).sort().join(",") : undefined; + return match ? columnList(match[1]!) : undefined; +} + +/** + * An index predicate in the form the schema check compares: lower case, with + * no parentheses or white space. Postgres stores `a IS NOT NULL AND b IS NULL` + * back as `((a IS NOT NULL) AND (b IS NULL))`; both reduce to the same text. + */ +export function comparablePredicate(predicate: string): string { + return predicate.toLowerCase().replace(/[()\s]/g, ""); } /* @@ -186,35 +200,49 @@ export const REQUIRED_RELATIONS = [ ...schemaText.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), ].map((match) => match[1]!); -/** Every table the schema creates. The store reads and writes each one. */ -export const REQUIRED_TABLES = createdTables.map(({ table }) => table); - -/** What the store does to its tables: it reads, inserts and updates, and never deletes. */ -export const REQUIRED_TABLE_PRIVILEGES = ["SELECT", "INSERT", "UPDATE"]; - -/** Every column a created table declares, and every column the schema adds to an existing table. */ +/** + * Every column a created table declares and every column the schema adds to + * an existing table, each with its declared type: the store reads and writes + * the values as that type, and a column retyped under it fails its queries. + */ export const REQUIRED_COLUMNS = [ ...createdTables.flatMap(({ table, lines }) => lines .filter((line) => !/^(?:PRIMARY KEY|UNIQUE)\b/.test(line)) - .map((line) => ({ table, column: /^(\w+)/.exec(line)![1]! })) + .map((line) => { + const [, column, type] = /^(\w+) (\w+)/.exec(line)!; + return { table, column: column!, type: type! }; + }) ), - ...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm)].map((match) => ({ + ...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+) (\w+)/gm)].map((match) => ({ table: match[1]!, column: match[2]!, + type: match[3]!, })), ]; /** - * Every primary key and unique constraint a created table declares, by its - * sorted columns. Each must stand as a unique, non-partial index on exactly - * those columns: the store's ON CONFLICT clauses and the foreign keys need - * nothing more, and without it every such write fails. + * Every unique key the schema makes: the primary keys and unique constraints + * of the created tables, and each unique index it creates by name with that + * index's predicate. The store relies on each as an immediate unique index + * whose key columns are exactly these, with exactly this predicate: ON + * CONFLICT refuses a deferrable arbiter and cannot infer a partial or wider + * one, and the partial attempt index is what keeps one unconsumed completion + * per attempt (R21.1). `index` is empty for a constraint, which may be + * satisfied by any such index on its table. */ -export const REQUIRED_KEYS = createdTables.flatMap(({ table, lines }) => - lines.flatMap((line) => { - const columns = - /^(\w+) .*\bPRIMARY KEY\b/.exec(line)?.[1] ?? keyColumns(line, "PRIMARY KEY") ?? keyColumns(line, "UNIQUE"); - return columns ? [{ table, columns }] : []; - }) -); +export const REQUIRED_KEYS = [ + ...createdTables.flatMap(({ table, lines }) => + lines.flatMap((line) => { + const columns = + /^(\w+) .*\bPRIMARY KEY\b/.exec(line)?.[1] ?? keyColumns(line, "PRIMARY KEY") ?? keyColumns(line, "UNIQUE"); + return columns ? [{ table, columns, index: "", predicate: "" }] : []; + }) + ), + ...[...schemaText.matchAll(/^CREATE UNIQUE INDEX IF NOT EXISTS (\w+)\s+ON (\w+)\(([^)]*)\)(?:\s+WHERE ([^;]+))?;/gm)].map((match) => ({ + table: match[2]!, + columns: columnList(match[3]!), + index: match[1]!, + predicate: comparablePredicate(match[4] ?? ""), + })), +]; diff --git a/tests/store.test.ts b/tests/store.test.ts index b208956..ffec529 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -1,17 +1,11 @@ import { randomUUID } from "node:crypto"; import { Pool } from "pg"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { newDb } from "pg-mem"; import { type HeimdallConfig } from "../src/config.js"; import { createStore } from "../src/store/index.js"; -import { createPostgresStore, PostgresStore } from "../src/store/postgres.js"; -import { - REQUIRED_COLUMNS, - REQUIRED_KEYS, - REQUIRED_RELATIONS, - REQUIRED_TABLE_PRIVILEGES, - REQUIRED_TABLES, -} from "../src/store/schema.js"; +import { createPostgresStore, PostgresStore, STORE_TABLE_PRIVILEGES } from "../src/store/postgres.js"; +import { comparablePredicate, REQUIRED_COLUMNS, REQUIRED_KEYS, REQUIRED_RELATIONS } from "../src/store/schema.js"; describe("PostgresStore", () => { it("round-trips core auth records through postgres", async () => { @@ -222,44 +216,41 @@ describe("createPostgresStore", () => { }); describe("PostgresStore.checkSchema", () => { - it("requires every table, index, column and key the schema makes", () => { + it("derives every relation, typed column and unique key from the schema", () => { expect(REQUIRED_RELATIONS).toEqual( expect.arrayContaining(["accounts", "audit_events", "auth_completions_attempt_unconsumed_unique_idx", "audit_events_lookup_idx"]) ); expect(REQUIRED_RELATIONS).toHaveLength(17); - expect(REQUIRED_TABLES).toEqual([ - "accounts", - "linked_identities", - "sessions", - "auth_attempts", - "private_command_receipts", - "auth_completions", - "capability_grants", - "entitlement_snapshots", - "audit_events", - ]); expect(REQUIRED_COLUMNS).toHaveLength(85); expect(REQUIRED_COLUMNS).toEqual( expect.arrayContaining([ - { table: "auth_completions", column: "code" }, - { table: "auth_completions", column: "mode" }, - { table: "linked_identities", column: "profile_json" }, - { table: "auth_completions", column: "attempt_id" }, + { table: "auth_completions", column: "code", type: "TEXT" }, + { table: "sessions", column: "access_revision", type: "INTEGER" }, + { table: "linked_identities", column: "profile_json", type: "JSONB" }, + { table: "entitlement_snapshots", column: "is_allowed", type: "BOOLEAN" }, + { table: "auth_completions", column: "attempt_id", type: "TEXT" }, ]) ); - // The keys the store's ON CONFLICT clauses and the foreign keys rely on. + expect(REQUIRED_KEYS).toHaveLength(12); expect(REQUIRED_KEYS).toEqual( expect.arrayContaining([ - { table: "linked_identities", columns: "provider,provider_user_id" }, - { table: "sessions", columns: "id" }, - { table: "private_command_receipts", columns: "app_slug,idempotency_key" }, - { table: "entitlement_snapshots", columns: "account_id,provider,scope" }, - { table: "accounts", columns: "id" }, - { table: "auth_completions", columns: "code" }, + { table: "linked_identities", columns: "provider,provider_user_id", index: "", predicate: "" }, + { table: "private_command_receipts", columns: "app_slug,idempotency_key", index: "", predicate: "" }, + { table: "entitlement_snapshots", columns: "account_id,provider,scope", index: "", predicate: "" }, + { + table: "auth_completions", + columns: "app_slug,attempt_id", + index: "auth_completions_attempt_unconsumed_unique_idx", + predicate: comparablePredicate("attempt_id IS NOT NULL AND consumed_at IS NULL"), + }, ]) ); - expect(REQUIRED_KEYS).toHaveLength(11); - expect(REQUIRED_TABLE_PRIVILEGES).toEqual(["SELECT", "INSERT", "UPDATE"]); + }); + + it("compares a predicate in the form Postgres stores it back", () => { + expect(comparablePredicate("((attempt_id IS NOT NULL) AND (consumed_at IS NULL))")).toBe( + comparablePredicate("attempt_id IS NOT NULL AND consumed_at IS NULL") + ); }); function storeAnswering(kinds: string[]) { @@ -282,16 +273,92 @@ describe("PostgresStore.checkSchema", () => { }); }); +/** + * Every operation the store offers, once, in an order that makes each one + * meaningful. Returns the names it called, so a test can prove it covered the + * store. Throws on the first operation the database refuses. + */ +async function exerciseStore(store: PostgresStore): Promise> { + const called = new Set(); + const s = new Proxy(store, { + get(target, name, receiver) { + const value = Reflect.get(target, name, receiver) as unknown; + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + called.add(String(name)); + return (value as (...a: unknown[]) => unknown).apply(target, args); + }; + }, + }); + const now = "2026-04-26T12:00:00.000Z"; + const later = "2999-01-01T00:00:00.000Z"; + const account = await s.createAccount({ createdAt: now, lastSeenAt: now, displayName: "Probe" }); + await s.touchAccount(account.id, now, { displayName: "Probe 2" }); + await s.findAccountById(account.id); + const identity = { + accountId: account.id, + provider: "discord" as const, + providerUserId: "u1", + scopes: ["identify"], + profileJson: {}, + createdAt: now, + updatedAt: now, + }; + await s.upsertLinkedIdentity(identity); + await s.upsertLinkedIdentity({ ...identity, username: "again" }); + await s.findAccountByLinkedIdentity("discord", "u1"); + await s.findStoredLinkedIdentity("discord", "u1"); + await s.findStoredLinkedIdentityForAccount(account.id, "discord"); + await s.listStoredLinkedIdentitiesForAccount(account.id); + await s.listLinkedIdentitiesForAccount(account.id); + await s.createCapabilityGrant({ + accountId: account.id, + scopeType: "app", + scopeId: "ghostlight", + capability: "app_access", + source: "probe", + status: "active", + createdAt: now, + updatedAt: now, + }); + await s.listActiveGrants(account.id, "ghostlight", now); + const session = { id: randomUUID(), accountId: account.id, appSlug: "ghostlight" as const, createdAt: now, lastSeenAt: now, expiresAt: later, accessRevision: 1, claimsJson: {} }; + await s.createSession(session); + await s.createSession({ ...session, lastSeenAt: later }); + await s.findSession("ghostlight", session.id); + await s.revokeSession("ghostlight", session.id, account.id, 1, now); + const attempt = await s.createAuthAttempt({ appSlug: "ghostlight", provider: "discord", mode: "sign_in", returnTo: "https://x.test/", createdAt: now, expiresAt: later }); + await s.findAuthAttempt("ghostlight", attempt.handle); + await s.updateAuthAttempt("ghostlight", attempt.handle, { status: "completed", at: now }); + const receipt = { appSlug: "ghostlight" as const, idempotencyKey: "k", requestFingerprint: "f", status: "accepted", contentSchema: "c", envelopeBase64: "e", createdAt: now, expiresAt: later }; + await s.createPrivateCommandReceipt(receipt); + await s.createPrivateCommandReceipt(receipt); + await s.findPrivateCommandReceipt("ghostlight", "k"); + const completion = { appSlug: "ghostlight" as const, provider: "discord" as const, mode: "sign_in" as const, accountId: account.id, sessionId: session.id, returnTo: "https://x.test/", createdAt: now, expiresAt: later, payloadJson: {} }; + const byCode = await s.createAuthCompletion(completion); + await s.consumeAuthCompletion("ghostlight", byCode.code, now); + await s.createAuthCompletion({ ...completion, attemptId: attempt.handle }); + await s.consumeAuthCompletionByAttempt("ghostlight", attempt.handle, now); + const snapshot = { accountId: account.id, provider: "discord" as const, scope: "ghostlight:probe", evaluatedAt: now, isAllowed: true, reasonCode: "ok", rawSummaryJson: {} }; + await s.upsertEntitlementSnapshot(snapshot); + await s.upsertEntitlementSnapshot({ ...snapshot, isAllowed: false }); + await s.createAuditEvent({ eventType: "probe", eventPayloadJson: {}, createdAt: now }); + return called; +} + // Against a real Postgres, through the startup path itself: createStore, with // and without applying the schema. HEIMDALL_TEST_PG_ADMIN_URL names a role -// that may create databases and roles; each case gets its own database. +// that may create databases and roles. Each case clones a template database +// that already holds the schema, and runs as its own login role. const adminUrl = process.env.HEIMDALL_TEST_PG_ADMIN_URL; -describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_ADMIN_URL)", () => { +describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_ADMIN_URL)", { timeout: 120_000 }, () => { const refusal = (code: string) => `Postgres storage could not be prepared (${code}); check GC_ACCESS_DATABASE_URL_FILE.`; + const suffix = randomUUID().replace(/-/g, "").slice(0, 12); + const template = `heimdall_template_${suffix}`; + const role = `heimdall_app_${suffix}`; + const rolePassword = "CANARYrolepw"; + const admin = adminUrl ? new Pool({ connectionString: adminUrl, max: 2 }) : undefined; const cleanup: Array<() => Promise> = []; - afterEach(async () => { - for (const step of cleanup.splice(0).reverse()) await step(); - }); function urlFor(database: string, user?: string, password?: string): string { const url = new URL(adminUrl!); @@ -301,104 +368,174 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD return url.toString(); } - /** A fresh database with the schema applied by its owner, and an owner pool on it. */ - async function appliedDatabase(): Promise<{ name: string; pool: Pool }> { + beforeAll(async () => { + await admin!.query(`CREATE DATABASE ${template}`); + const pool = new Pool({ connectionString: urlFor(template) }); + await new PostgresStore(pool).ensureSchema(); + await pool.end(); + await admin!.query(`CREATE ROLE ${role} LOGIN PASSWORD '${rolePassword}'`); + }, 120_000); + + afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); + }); + + afterAll(async () => { + await admin!.query(`DROP DATABASE IF EXISTS ${template}`); + await admin!.query(`DROP ROLE IF EXISTS ${role}`); + await admin!.end(); + }, 120_000); + + /** A clone of the applied template, and a pool on it as its owner (the admin role). */ + async function appliedDatabase(): Promise<{ name: string; owner: Pool }> { const name = `heimdall_check_${randomUUID().replace(/-/g, "")}`; - const admin = new Pool({ connectionString: adminUrl }); - await admin.query(`CREATE DATABASE ${name}`); - cleanup.push(async () => { - await admin.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); - await admin.end(); - }); - const pool = new Pool({ connectionString: urlFor(name) }); + await admin!.query(`CREATE DATABASE ${name} TEMPLATE ${template}`); + const owner = new Pool({ connectionString: urlFor(name), max: 2 }); // The forced drop ends this fixture pool's connections, possibly while // they are still closing; that is cleanup, not a finding. - pool.on("error", () => undefined); + owner.on("error", () => undefined); cleanup.push(async () => { - await pool.end(); + await owner.end(); + await admin!.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); }); - await new PostgresStore(pool).ensureSchema(); - return { name, pool }; + return { name, owner }; } + /** Grants the app role exactly `privileges`, per table, and nothing else. */ + async function grant(owner: Pool, privileges: Readonly>): Promise { + await owner.query(`GRANT USAGE ON SCHEMA public TO ${role}`); + for (const [table, wanted] of Object.entries(privileges)) { + if (wanted.length > 0) await owner.query(`GRANT ${wanted.join(", ")} ON ${table} TO ${role}`); + } + } + + /** + * Starts Heimdall's store on `url` and reports "started" or the refusal's + * message. A refusal must also leave no session of its own behind: the + * refused store's pool is closed before the error is thrown. + */ async function start(url: string, applySchemaOnStartup: boolean): Promise { + const tagged = new URL(url); + const tag = `heimdall_start_${randomUUID().replace(/-/g, "").slice(0, 16)}`; + tagged.searchParams.set("application_name", tag); try { - const store = await createStore({ storage: { backend: "postgres", databaseUrl: url, applySchemaOnStartup } } as HeimdallConfig); + const store = await createStore({ + storage: { backend: "postgres", databaseUrl: tagged.toString(), applySchemaOnStartup }, + } as HeimdallConfig); await store.close(); return "started"; } catch (error) { + let sessions = -1; + for (let attempt = 0; attempt < 50 && sessions !== 0; attempt += 1) { + if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, 100)); + const result = await admin!.query<{ n: number }>( + "SELECT count(*)::int AS n FROM pg_stat_activity WHERE application_name = $1", + [tag] + ); + sessions = result.rows[0]!.n; + } + expect(sessions, "a refused store left its connection open").toBe(0); return (error as Error).message; } } - it("starts on an applied database, applying or not", async () => { + it("starts on an applied database as its owner, applying or not", async () => { const { name } = await appliedDatabase(); expect(await start(urlFor(name), false)).toBe("started"); expect(await start(urlFor(name), true)).toBe("started"); }); - it("refuses a role that may only read, by code and without its password", async () => { - const { name, pool } = await appliedDatabase(); - const role = `heimdall_ro_${randomUUID().replace(/-/g, "").slice(0, 12)}`; - await pool.query(`CREATE ROLE ${role} LOGIN PASSWORD 'CANARYrolepw'`); + it("serves every store operation with exactly the privileges it checks for", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, STORE_TABLE_PRIVILEGES); + + expect(await start(urlFor(name, role, rolePassword), false)).toBe("started"); + const pool = new Pool({ connectionString: urlFor(name, role, rolePassword) }); cleanup.push(async () => { - await pool.query(`REVOKE ALL ON ALL TABLES IN SCHEMA public FROM ${role}`); - await pool.query(`DROP ROLE IF EXISTS ${role}`); + await pool.end(); }); - await pool.query(`GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${role}`); + const called = await exerciseStore(new PostgresStore(pool)); + const offered = Object.getOwnPropertyNames(PostgresStore.prototype).filter( + (method) => !["constructor", "ensureSchema", "checkSchema", "close"].includes(method) + ); + expect([...called].sort()).toEqual(offered.sort()); + }); - const message = await start(urlFor(name, role, "CANARYrolepw"), false); - expect(message).toBe(refusal("SCHEMA_PRIVILEGES")); - expect(message).not.toContain("CANARY"); + it("refuses each privilege taken away, and the store needs each one", async () => { + const pairs = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => wanted.map((privilege) => ({ table, privilege }))); + expect(Object.keys(STORE_TABLE_PRIVILEGES).sort()).toEqual( + REQUIRED_RELATIONS.filter((relation) => REQUIRED_COLUMNS.some((column) => column.table === relation)).sort() + ); + for (const { table, privilege } of pairs) { + const { name, owner } = await appliedDatabase(); + await grant(owner, { ...STORE_TABLE_PRIVILEGES, [table]: STORE_TABLE_PRIVILEGES[table]!.filter((p) => p !== privilege) }); - // The refused store closed its pool: no session of the role outlives the refusal. - let sessions = -1; - for (let attempt = 0; attempt < 20 && sessions !== 0; attempt += 1) { - if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, 100)); - const result = await pool.query<{ n: number }>("SELECT count(*)::int AS n FROM pg_stat_activity WHERE usename = $1", [role]); - sessions = result.rows[0]!.n; + const message = await start(urlFor(name, role, rolePassword), false); + expect(message, `${table} without ${privilege}`).toBe(refusal("SCHEMA_PRIVILEGES")); + expect(message).not.toContain("CANARY"); + + const pool = new Pool({ connectionString: urlFor(name, role, rolePassword) }); + const failure = await exerciseStore(new PostgresStore(pool)).then( + () => undefined, + (error: unknown) => (error as { code?: string }).code + ); + await pool.end(); + expect(failure, `the store runs without ${privilege} on ${table}, so the check need not ask for it`).toBe("42501"); + for (const step of cleanup.splice(0).reverse()) await step(); } - expect(sessions).toBe(0); + }); + + it("refuses to apply the schema as a role that may not, and closes the store", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, STORE_TABLE_PRIVILEGES); + + expect(await start(urlFor(name, role, rolePassword), true)).toBe(refusal("42501")); }); it("refuses a table whose primary key was dropped, even after applying the schema", async () => { - const { name, pool } = await appliedDatabase(); - await pool.query("ALTER TABLE auth_completions DROP CONSTRAINT auth_completions_pkey"); + const { name, owner } = await appliedDatabase(); + await owner.query("ALTER TABLE auth_completions DROP CONSTRAINT auth_completions_pkey"); expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); expect(await start(urlFor(name), true)).toBe(refusal("SCHEMA_MISSING")); }); - it("refuses a unique key the store's ON CONFLICT needs, replaced by a partial one", async () => { - const { name, pool } = await appliedDatabase(); - await pool.query("ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key"); - await pool.query( - "CREATE UNIQUE INDEX entitlement_snapshots_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed" + // Each damage below leaves a database whose shape still resembles the + // schema but on which a store write fails or R21.1 no longer holds. + it.each([ + ["a key made DEFERRABLE (ON CONFLICT refuses it as an arbiter)", + "ALTER TABLE private_command_receipts DROP CONSTRAINT private_command_receipts_pkey; ALTER TABLE private_command_receipts ADD PRIMARY KEY (app_slug, idempotency_key) DEFERRABLE INITIALLY IMMEDIATE"], + ["a key narrowed to one column, the other only INCLUDEd", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_include ON linked_identities(provider) INCLUDE (provider_user_id)"], + ["a key widened by an expression column", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_expr ON linked_identities(provider, provider_user_id, lower(username))"], + ["a key replaced by a partial one", + "ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key; CREATE UNIQUE INDEX es_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed"], + ["a key replaced by a plain index on the same columns", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE INDEX li_plain ON linked_identities(provider, provider_user_id)"], + ["a key replaced by one on other columns of the same count", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_other ON linked_identities(provider, username)"], + ["the attempt index replaced by a plain one of the same name", + "DROP INDEX auth_completions_attempt_unconsumed_unique_idx; CREATE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id)"], + ["the attempt index made unique on every row (no predicate)", + "DROP INDEX auth_completions_attempt_unconsumed_unique_idx; CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id)"], + ["the attempt index given another predicate", + "DROP INDEX auth_completions_attempt_unconsumed_unique_idx; CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE attempt_id IS NOT NULL"], + ["a column retyped", "ALTER TABLE sessions ALTER COLUMN access_revision TYPE TEXT"], + ["a created table's column dropped", "ALTER TABLE auth_completions DROP COLUMN mode"], + ])("refuses %s", async (_damage, sql) => { + const { name, owner } = await appliedDatabase(); + await owner.query(sql); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + }); + + it("accepts a key index that carries extra INCLUDE columns", async () => { + const { name, owner } = await appliedDatabase(); + await owner.query( + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_covering ON linked_identities(provider, provider_user_id) INCLUDE (username)" ); - expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); - }); - - it("refuses a unique key replaced by a plain index on the same columns", async () => { - const { name, pool } = await appliedDatabase(); - await pool.query("ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key"); - await pool.query("CREATE INDEX linked_identities_plain ON linked_identities(provider, provider_user_id)"); - - expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); - }); - - it("refuses a unique key replaced by one on other columns of the same count", async () => { - const { name, pool } = await appliedDatabase(); - await pool.query("ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key"); - await pool.query("CREATE UNIQUE INDEX linked_identities_other ON linked_identities(provider, username)"); - - expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); - }); - - it("refuses a created table's dropped column", async () => { - const { name, pool } = await appliedDatabase(); - await pool.query("ALTER TABLE auth_completions DROP COLUMN mode"); - - expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + expect(await start(urlFor(name), false)).toBe("started"); }); }); From 963150f537a64a6fd1d79342bc553568c8e7430d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:56:35 +0200 Subject: [PATCH 32/85] Audit events never carry a completion code Soul pass 5, S5-3: auth_completion_created stored completion.code, which the unauthenticated redeem route turns into the session until it expires (300 s by default), so the audit trail could redeem a live sign-in. The two auth_completion_redeemed events (the HTTP redeem route and the private command plane's redeem by attempt) stored the spent code as well. All three payloads drop it. Nothing reads audit payloads back: the store has no audit read, and no repository indexed by VoidBot consumes these events. The event's sessionId already names the completion (one per callback), so no replacement key is added. This matches Heimdall's own witness contract, which says codes are redacted from audit exports. Tests drive a sign-in through the HTTP redeem route and through the plane, pin each payload exactly, and check the code appears in no audit event. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 5 +-- src/private-command-plane.ts | 1 - tests/app.test.ts | 27 ++++++++++++++++ tests/private-command-diagnostics.test.ts | 39 +++++++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/app.ts b/src/app.ts index eaaef7d..9621856 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1304,10 +1304,12 @@ export async function buildApp(options: BuildAppOptions = {}): Promise { expect(secondRedeemResponse.statusCode).toBe(410); }); + it("keeps the completion code out of the audit trail, live or redeemed", async () => { + const store = new InMemoryStore(); + const app = await buildApp({ config: createTestConfig(), store, oauthRuntimes: { discord: createMockDiscordRuntime() } }); + apps.push(app); + const auditEvents = () => + [...(store as unknown as { auditEvents: Map }).auditEvents.values()]; + + const stateToken = await startDiscordSignIn(app); + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=test-code&state=${encodeURIComponent(stateToken)}`, + }); + const { code, expiresAt } = callback.json().completion as { code: string; expiresAt: string }; + const created = auditEvents().filter((event) => event.eventType === "auth_completion_created"); + expect(created.map((event) => event.eventPayloadJson)).toEqual([{ provider: "discord", mode: "sign_in", expiresAt }]); + expect(created[0]!.sessionId).toEqual(expect.any(String)); + // While the code is live, the audit trail must not be a way to redeem it. + expect(JSON.stringify(auditEvents())).not.toContain(code); + + const redeem = await app.inject({ method: "POST", url: "/v1/apps/repixelizer/auth-completions/redeem", payload: { completionCode: code } }); + expect(redeem.statusCode).toBe(201); + const redeemed = auditEvents().filter((event) => event.eventType === "auth_completion_redeemed"); + expect(redeemed.map((event) => event.eventPayloadJson)).toEqual([{ provider: "discord", mode: "sign_in" }]); + expect(redeemed[0]!.sessionId).toBe(created[0]!.sessionId); + expect(JSON.stringify(auditEvents())).not.toContain(code); + }); + it("refreshes a Repixelizer app claim from a Heimdall refresh token without provider OAuth", async () => { const store = new InMemoryStore(); let exchangeCalls = 0; diff --git a/tests/private-command-diagnostics.test.ts b/tests/private-command-diagnostics.test.ts index 3204e10..4f7e965 100644 --- a/tests/private-command-diagnostics.test.ts +++ b/tests/private-command-diagnostics.test.ts @@ -109,3 +109,42 @@ it("keeps a provider's refusal body out of every plaintext part of the reply", a expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ProviderResponseError, provider_status)."]); expect(plaintext(reply)).not.toContain("CANARY"); }); + +it("keeps the completion code out of the audit trail when the plane redeems by attempt", async () => { + vi.stubGlobal("fetch", async (url: string | URL) => { + const u = String(url); + if (u.endsWith("/oauth2/token")) { + return new Response( + JSON.stringify({ access_token: "a", refresh_token: "r", token_type: "Bearer", expires_in: 60, scope: "identify" }), + { status: 200 } + ); + } + if (u.endsWith("/users/@me")) return new Response(JSON.stringify({ id: "u1", username: "u1" }), { status: 200 }); + if (u.includes("/member")) return new Response(JSON.stringify({ roles: ["r"] }), { status: 200 }); + return new Response("{}", { status: 404 }); + }); + const cfg = config(); + const store = new InMemoryStore(); + const app = await buildApp({ config: cfg, store }); + resources.push(app); + const plane = await startHeimdallPrivateCommandPlane(app, cfg); + resources.push(plane); + const policy = { kind: "discord_role_access", guildId: "g", allowedRoleIds: ["r"] }; + const begin = open(await command(plane.endpoint, "heimdall.auth.begin", "heimdall.auth_begin_command.v1", "k1", + { provider: "discord", mode: "sign_in", returnTo: "https://yggdrasil.gamecult.org/ghostlight/", entitlementPolicy: policy })); + const state = new URL(String((begin.navigation as { url: string }).url)).searchParams.get("state")!; + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=c1&state=${encodeURIComponent(state)}`, + headers: { accept: "application/json" }, + }); + const code = (callback.json().completion as { code: string }).code; + const done = open(await command(plane.endpoint, "heimdall.auth.complete", "heimdall.auth_complete_command.v1", "k2", { handle: begin.handle })); + expect(done.status).toBe("authenticated"); + + const events = [...(store as unknown as { auditEvents: Map }).auditEvents.values()]; + expect(events.filter((event) => event.eventType === "auth_completion_redeemed").map((event) => event.eventPayloadJson)).toEqual([ + { provider: "discord", mode: "sign_in" }, + ]); + expect(JSON.stringify(events)).not.toContain(code); +}); From 35f804f7ef6d0439551c59883e15cb2b5407ea80 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:57:17 +0200 Subject: [PATCH 33/85] Pin provider errors and Bifrost's reply exactly; defend errorIdentity's filters Soul pass 5, S5-4: the provider-error tests proved only that the word CANARY was absent, so a message carrying a 16-byte slice of the body (X1, X2) passed, and the Bifrost success reply was checked with objectContaining, so a 9-byte body head (X5) rode along. Each provider error's message is now pinned exactly, the Bifrost success reply is pinned as exactly status, supportFact and bifrostStatus, and every canary body starts with the canary, so any prefix of it shows. S5-5: errorIdentity's class and code filters had no test (X3, X4). Cases now show a code or class name that carries input, is too long, is not a string, or comes from a thrown non-Error or null is dropped, and a short identifier survives. Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 44 ++++++++++++++++++----- tests/private-command-diagnostics.test.ts | 2 +- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/app.test.ts b/tests/app.test.ts index bdf0711..044191a 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { type FastifyInstance } from "fastify"; -import { buildApp } from "../src/app.js"; +import { buildApp, errorIdentity } from "../src/app.js"; import { deliverBackendHandoff, type BackendHandoffPayload } from "../src/backend-handoff.js"; import { type HeimdallConfig, loadConfig } from "../src/config.js"; import { entitlementFacts, identityFacts } from "../src/facts.js"; @@ -614,7 +614,7 @@ describe("Heimdall service", () => { it("syncs a linked Patreon membership into a signed Bifrost support fact", async () => { const { response, auditEvents, bifrostBody, bifrostSignature, now } = await syncPatronSupport( - () => new Response("processed CANARYbifrost", { status: 200 }) + () => new Response("CANARYbifrost processed", { status: 200 }) ); expect(response.statusCode, response.body).toBe(200); @@ -633,7 +633,8 @@ describe("Heimdall service", () => { providerSubscriptionId: "member-789", notes: "Verified active Patreon membership for tier Inner Sanctum.", }); - expect(response.json()).toEqual(expect.objectContaining({ status: "synced", bifrostStatus: 200 })); + // Exactly these fields: no part of Bifrost's body rides along. + expect(response.json()).toEqual({ status: "synced", supportFact: JSON.parse(bifrostBody), bifrostStatus: 200 }); expect(auditEvents.map((event) => [event.eventType, event.eventPayloadJson])).toEqual([ [ "bifrost_patron_support_synced", @@ -650,7 +651,7 @@ describe("Heimdall service", () => { it("answers a Bifrost refusal with fixed text and its status, never its body", async () => { const { response, auditEvents } = await syncPatronSupport( - () => new Response('{"error":"bad signature","echo":"CANARYrefusal"}', { status: 403 }) + () => new Response('CANARYrefusal {"error":"bad signature"}', { status: 403 }) ); expect(response.statusCode).toBe(502); @@ -1604,7 +1605,7 @@ describe("the failed-callback audit event for a provider's HTTP refusal", () => it("keeps the provider's status as a number and never its body", async () => { const failed = await auditAfterTokenExchangeAnswers( - () => new Response('{"error":"invalid_grant","echo":"CANARYbody"}', { status: 401 }) + () => new Response('CANARYbody {"error":"invalid_grant"}', { status: 401 }) ); expect(failed).toHaveLength(1); @@ -1620,7 +1621,7 @@ describe("the failed-callback audit event for a provider's HTTP refusal", () => it("keeps a 2xx body that is not JSON out of the event, by class and code alone", async () => { const failed = await auditAfterTokenExchangeAnswers( - () => new Response('{"access_token":"CANARYaccess","refresh_token":CANARYrefresh}', { status: 200 }) + () => new Response('CANARYaccess{"access_token":"a"}', { status: 200 }) ); expect(failed).toHaveLength(1); @@ -1660,7 +1661,7 @@ describe("the failed-callback audit event for a provider's HTTP refusal", () => describe("a backend handoff the app refuses", () => { it("fails by status alone, never with the app's answer body", async () => { - globalThis.fetch = async () => new Response('{"error":"rejected","echo":"CANARYhandoff"}', { status: 500 }); + globalThis.fetch = async () => new Response('CANARYhandoff {"error":"rejected"}', { status: 500 }); const failure = await deliverBackendHandoff("https://bifrost.gamecult.org/auth/heimdall/callback", { source: "heimdall", kind: "oauth_result", @@ -1685,8 +1686,9 @@ describe("a backend handoff the app refuses", () => { describe("a provider answer Heimdall cannot use", () => { const runtimes = createOAuthRuntimeRegistry(); - const notJson = '{"access_token":"CANARYaccess","refresh_token":CANARYrefresh}'; - const refusal = '{"error":"invalid_grant","echo":"CANARYrefused"}'; + // The canary leads each body, so a message carrying any prefix of it shows. + const notJson = 'CANARYaccess{"access_token":"a"}'; + const refusal = 'CANARYrefused {"error":"invalid_grant"}'; /** What the call threw, as every form a catcher might record or print. */ async function failureOf(call: () => Promise) { @@ -1711,6 +1713,7 @@ describe("a provider answer Heimdall cannot use", () => { ); expect(error.name).toBe("ProviderResponseError"); + expect(error.message).toBe(`Discord token exchange failed (status ${status}, ${code}).`); expect(error.code).toBe(code); expect(error.status).toBe(status); expect(printed).not.toContain("CANARY"); @@ -1732,6 +1735,7 @@ describe("a provider answer Heimdall cannot use", () => { }) ); + expect(error.message).toBe("Discord guild member lookup failed (status 200, provider_body_not_json)."); expect(error.code).toBe("provider_body_not_json"); expect(printed).not.toContain("CANARY"); }); @@ -1742,7 +1746,29 @@ describe("a provider answer Heimdall cannot use", () => { runtimes.youtube.resolveIdentity({ config: createTestConfig(), accessToken: "a" }) ); + expect(error.message).toBe("YouTube channel lookup failed (status 200, provider_body_not_json)."); expect(error.code).toBe("provider_body_not_json"); expect(printed).not.toContain("CANARY"); }); }); + +describe("what may be recorded about a caught error", () => { + it("keeps a short identifier class and code", () => { + expect(errorIdentity(Object.assign(new TypeError("CANARY message"), { code: "ERR_INVALID_URL" }))).toEqual({ + errorClass: "TypeError", + errorCode: "ERR_INVALID_URL", + }); + }); + + it.each([ + ["a code that carries input", Object.assign(new Error("m"), { code: "CANARY host.invalid:5432" }), { errorClass: "Error", errorCode: null }], + ["a code longer than an identifier", Object.assign(new Error("m"), { code: `CANARY${"x".repeat(64)}` }), { errorClass: "Error", errorCode: null }], + ["a code that is not a string", Object.assign(new Error("m"), { code: 42 }), { errorClass: "Error", errorCode: null }], + ["a class name that carries input", Object.assign(new Error("m"), { name: "CANARY: postgres://u:p@h/db" }), { errorClass: "Error", errorCode: null }], + ["a class name longer than an identifier", Object.assign(new Error("m"), { name: `CANARY${"x".repeat(64)}` }), { errorClass: "Error", errorCode: null }], + ["a thrown value that is not an Error", { name: "CANARYname", code: "CANARY code" }, { errorClass: "Error", errorCode: null }], + ["a thrown null", null, { errorClass: "Error", errorCode: null }], + ])("filters %s", (_case, error, expected) => { + expect(errorIdentity(error)).toEqual(expected); + }); +}); diff --git a/tests/private-command-diagnostics.test.ts b/tests/private-command-diagnostics.test.ts index 4f7e965..41f16f1 100644 --- a/tests/private-command-diagnostics.test.ts +++ b/tests/private-command-diagnostics.test.ts @@ -103,7 +103,7 @@ it("keeps a provider's malformed 2xx refresh body out of every plaintext part of }); it("keeps a provider's refusal body out of every plaintext part of the reply", async () => { - const reply = await refreshAgainst(() => new Response('{"error":"invalid_grant","echo":"CANARYrefused"}', { status: 400 })); + const reply = await refreshAgainst(() => new Response('CANARYrefused {"error":"invalid_grant"}', { status: 400 })); expect(reply.status).toBe("denied"); expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ProviderResponseError, provider_status)."]); From fd98e36186b177199f2eb60b1d4d60a3fc46c118 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 18:58:02 +0200 Subject: [PATCH 34/85] Recipe test: explore loadConfig's branches, refuse environment copies Soul pass 5, S5-7. The recording environment saw only the reads an empty environment makes, so a secret read on a branch a value opens (X6) was missed, and a read from a spread copy of the environment (X7) was invisible. loadConfig now runs over many environments until no run reads a new name: every secret found so far set in plaintext, then each name read so far set to each of five probe values, then all of them set to one probe value at once; a run that throws keeps the reads it made first. Copying or enumerating the environment (the Proxy's ownKeys and getOwnPropertyDescriptor traps) is recorded, and a test refuses it, since reads from a copy cannot be seen. HMAC joins the secret-shaped backstop pattern. Co-Authored-By: Claude Opus 5.5 --- tests/recipe-secrets.test.ts | 75 ++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/tests/recipe-secrets.test.ts b/tests/recipe-secrets.test.ts index ec477a0..31717c3 100644 --- a/tests/recipe-secrets.test.ts +++ b/tests/recipe-secrets.test.ts @@ -32,38 +32,87 @@ const everyDeclaredName = [...recipe.replace(/\r\n/g, "\n").matchAll(/^\w+_envir // up. readSecretInput looks up `NAME_FILE` for each secret, so every `_FILE` // lookup names one, however the call is spelled and whatever list computes // the name. The same recorder stands in for process.env during the call, so a -// read that bypasses the env argument is seen too. The environment is empty, -// so this sees the reads loadConfig makes when nothing is set; a secret read -// only on a branch some other variable opens would not be seen. -function namesLoadConfigReads(): Set { +// read that bypasses the env argument is seen too. +// +// A read made from a copy of the environment ({ ...env }, Object.entries) +// cannot be seen, so copying or enumerating it is recorded and fails below. +// +// A secret read only on a branch that some value opens is reached by running +// loadConfig over many environments: every secret found so far set in +// plaintext, then each name read so far set to each probe value in turn, then +// every such name set to one probe value at once, until no run reads a new +// name. A run that throws still counts the reads it made first. +const probeValues = ["1", "true", "/probe/path", "127.0.0.1:4100", "https://probe.test"]; + +function readsOf(values: Readonly>): { names: Set; enumerated: boolean } { const names = new Set(); + let enumerated = false; + const record = (name: string | symbol) => { + if (typeof name === "string") names.add(name); + }; const env = new Proxy({} as NodeJS.ProcessEnv, { get(_target, name) { - if (typeof name === "string") names.add(name); - return undefined; + record(name); + return typeof name === "string" ? values[name] : undefined; }, has(_target, name) { - if (typeof name === "string") names.add(name); - return false; + record(name); + return typeof name === "string" && name in values; + }, + ownKeys() { + enumerated = true; + return []; + }, + getOwnPropertyDescriptor() { + enumerated = true; + return undefined; }, }); const processEnv = process.env; process.env = env; try { loadConfig(env, []); + } catch { + // The reads before the refusal still count. } finally { process.env = processEnv; } - return names; + return { names, enumerated }; } +function exploreLoadConfig(): { names: Set; enumerated: boolean } { + const names = new Set(); + let enumerated = false; + const secretsOf = () => [...names].filter((name) => name.endsWith("_FILE")).map((name) => name.slice(0, -"_FILE".length)); + for (let grew = true; grew; ) { + const before = names.size; + const secrets = Object.fromEntries(secretsOf().map((name) => [name, "probe-secret"])); + const settable = [...names].filter((name) => !name.endsWith("_FILE") && !(name in secrets)); + const runs: Array> = [ + {}, + secrets, + ...settable.flatMap((name) => probeValues.map((value) => ({ ...secrets, [name]: value }))), + ...probeValues.map((value) => ({ ...secrets, ...Object.fromEntries(settable.map((name) => [name, value])) })), + ]; + for (const values of runs) { + const run = readsOf(values); + run.names.forEach((name) => names.add(name)); + enumerated ||= run.enumerated; + } + grew = names.size > before; + } + return { names, enumerated }; +} + +const explored = exploreLoadConfig(); + /** Secrets Heimdall reads through readSecretInput, by their plaintext name. */ -const secretNames = [...namesLoadConfigReads()] +const secretNames = [...explored.names] .filter((name) => name.endsWith("_FILE")) .map((name) => name.slice(0, -"_FILE".length)); const secretShaped = - /SECRET|PASSWORD|PASSPHRASE|TOKEN|API_KEY|PRIVATE_KEY|ENCRYPTION_KEY|SALT|CREDENTIAL|DATABASE_URL|_DSN|_PEM/; + /SECRET|PASSWORD|PASSPHRASE|TOKEN|API_KEY|PRIVATE_KEY|ENCRYPTION_KEY|SALT|CREDENTIAL|DATABASE_URL|_DSN|_PEM|HMAC/; /** Secret-shaped names that hold no secret, each for its own reason. */ const notSecrets = new Set([ @@ -86,6 +135,10 @@ describe("Idunn recipe secret declarations", () => { expect(plaintext).toEqual([]); }); + it("reads the environment only by name, never by copying or enumerating it", () => { + expect(explored.enumerated).toBe(false); + }); + it("sees every secret loadConfig reads, across the provider and app lists", () => { expect(secretNames).toEqual( expect.arrayContaining([ From 6c0c87669fcb4bc056fa1d79c9f1f130303bdd7d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 19:30:40 +0200 Subject: [PATCH 35/85] Schema check: a unique key is matched by shape, not name; quiet fixture pools The rule mutation that stopped checking a named unique index by its name survived, and rightly: an index of the same table, key columns, predicate, uniqueness and immediacy keeps R21.1 whatever it is called, and the named index's existence is already required as a relation. The name condition and REQUIRED_KEYS' index field are deleted. The live block failed 5 of 10 repeated runs with unhandled 57P01 errors: a test's own pool, already ended, still had connections closing when the forced drop terminated them, and nothing listened. Every pool the tests open now goes through fixturePool, which ignores those errors; the store's own pool keeps its listener and is not a fixture. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 6 ++---- src/store/schema.ts | 20 +++++++++----------- tests/store.test.ts | 29 ++++++++++++++++++----------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index ea47ca0..4ba0ca7 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -335,11 +335,10 @@ export class PostgresStore implements HeimdallStore { AND a.atttypid = to_regtype(required.type_name) ) UNION ALL - SELECT 'missing' FROM unnest($5::text[], $6::text[], $7::text[], $8::text[]) AS required(table_name, columns, index_name, predicate) + SELECT 'missing' FROM unnest($5::text[], $6::text[], $7::text[]) AS required(table_name, columns, predicate) WHERE NOT EXISTS ( SELECT 1 FROM pg_index i WHERE i.indrelid = to_regclass(required.table_name) - AND (required.index_name = '' OR i.indexrelid = to_regclass(required.index_name)) AND i.indisunique AND i.indimmediate AND coalesce(regexp_replace(lower(pg_get_expr(i.indpred, i.indrelid)), '[()[:space:]]', '', 'g'), '') = required.predicate @@ -352,7 +351,7 @@ export class PostgresStore implements HeimdallStore { ) = required.columns ) UNION ALL - SELECT 'privilege' FROM unnest($9::text[], $10::text[]) AS required(table_name, privilege) + SELECT 'privilege' FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) WHERE to_regclass(required.table_name) IS NOT NULL AND NOT has_table_privilege(to_regclass(required.table_name), required.privilege) `, @@ -363,7 +362,6 @@ export class PostgresStore implements HeimdallStore { REQUIRED_COLUMNS.map((required) => required.type), REQUIRED_KEYS.map((required) => required.table), REQUIRED_KEYS.map((required) => required.columns), - REQUIRED_KEYS.map((required) => required.index), REQUIRED_KEYS.map((required) => required.predicate), privileges.map((required) => required.table), privileges.map((required) => required.privilege), diff --git a/src/store/schema.ts b/src/store/schema.ts index f385031..91db7fc 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -223,26 +223,24 @@ export const REQUIRED_COLUMNS = [ /** * Every unique key the schema makes: the primary keys and unique constraints - * of the created tables, and each unique index it creates by name with that - * index's predicate. The store relies on each as an immediate unique index - * whose key columns are exactly these, with exactly this predicate: ON + * of the created tables, and each unique index it creates, with that index's + * predicate. The store relies on each as an immediate unique index on its + * table whose key columns are exactly these, with exactly this predicate: ON * CONFLICT refuses a deferrable arbiter and cannot infer a partial or wider * one, and the partial attempt index is what keeps one unconsumed completion - * per attempt (R21.1). `index` is empty for a constraint, which may be - * satisfied by any such index on its table. + * per attempt (R21.1). Any index with that shape serves, whatever its name. */ export const REQUIRED_KEYS = [ ...createdTables.flatMap(({ table, lines }) => lines.flatMap((line) => { const columns = /^(\w+) .*\bPRIMARY KEY\b/.exec(line)?.[1] ?? keyColumns(line, "PRIMARY KEY") ?? keyColumns(line, "UNIQUE"); - return columns ? [{ table, columns, index: "", predicate: "" }] : []; + return columns ? [{ table, columns, predicate: "" }] : []; }) ), - ...[...schemaText.matchAll(/^CREATE UNIQUE INDEX IF NOT EXISTS (\w+)\s+ON (\w+)\(([^)]*)\)(?:\s+WHERE ([^;]+))?;/gm)].map((match) => ({ - table: match[2]!, - columns: columnList(match[3]!), - index: match[1]!, - predicate: comparablePredicate(match[4] ?? ""), + ...[...schemaText.matchAll(/^CREATE UNIQUE INDEX IF NOT EXISTS \w+\s+ON (\w+)\(([^)]*)\)(?:\s+WHERE ([^;]+))?;/gm)].map((match) => ({ + table: match[1]!, + columns: columnList(match[2]!), + predicate: comparablePredicate(match[3] ?? ""), })), ]; diff --git a/tests/store.test.ts b/tests/store.test.ts index ffec529..25500c7 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -234,13 +234,12 @@ describe("PostgresStore.checkSchema", () => { expect(REQUIRED_KEYS).toHaveLength(12); expect(REQUIRED_KEYS).toEqual( expect.arrayContaining([ - { table: "linked_identities", columns: "provider,provider_user_id", index: "", predicate: "" }, - { table: "private_command_receipts", columns: "app_slug,idempotency_key", index: "", predicate: "" }, - { table: "entitlement_snapshots", columns: "account_id,provider,scope", index: "", predicate: "" }, + { table: "linked_identities", columns: "provider,provider_user_id", predicate: "" }, + { table: "private_command_receipts", columns: "app_slug,idempotency_key", predicate: "" }, + { table: "entitlement_snapshots", columns: "account_id,provider,scope", predicate: "" }, { table: "auth_completions", columns: "app_slug,attempt_id", - index: "auth_completions_attempt_unconsumed_unique_idx", predicate: comparablePredicate("attempt_id IS NOT NULL AND consumed_at IS NULL"), }, ]) @@ -360,6 +359,17 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD const admin = adminUrl ? new Pool({ connectionString: adminUrl, max: 2 }) : undefined; const cleanup: Array<() => Promise> = []; + /** + * A pool the test itself uses. The forced drops end its connections, + * possibly while pg is still closing them after end() resolved; that is + * cleanup, not a finding, so it is not left to become an unhandled error. + */ + function fixturePool(connectionString: string): Pool { + const pool = new Pool({ connectionString, max: 2 }); + pool.on("error", () => undefined); + return pool; + } + function urlFor(database: string, user?: string, password?: string): string { const url = new URL(adminUrl!); url.pathname = `/${database}`; @@ -370,7 +380,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD beforeAll(async () => { await admin!.query(`CREATE DATABASE ${template}`); - const pool = new Pool({ connectionString: urlFor(template) }); + const pool = fixturePool(urlFor(template)); await new PostgresStore(pool).ensureSchema(); await pool.end(); await admin!.query(`CREATE ROLE ${role} LOGIN PASSWORD '${rolePassword}'`); @@ -390,10 +400,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD async function appliedDatabase(): Promise<{ name: string; owner: Pool }> { const name = `heimdall_check_${randomUUID().replace(/-/g, "")}`; await admin!.query(`CREATE DATABASE ${name} TEMPLATE ${template}`); - const owner = new Pool({ connectionString: urlFor(name), max: 2 }); - // The forced drop ends this fixture pool's connections, possibly while - // they are still closing; that is cleanup, not a finding. - owner.on("error", () => undefined); + const owner = fixturePool(urlFor(name)); cleanup.push(async () => { await owner.end(); await admin!.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); @@ -450,7 +457,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD await grant(owner, STORE_TABLE_PRIVILEGES); expect(await start(urlFor(name, role, rolePassword), false)).toBe("started"); - const pool = new Pool({ connectionString: urlFor(name, role, rolePassword) }); + const pool = fixturePool(urlFor(name, role, rolePassword)); cleanup.push(async () => { await pool.end(); }); @@ -474,7 +481,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD expect(message, `${table} without ${privilege}`).toBe(refusal("SCHEMA_PRIVILEGES")); expect(message).not.toContain("CANARY"); - const pool = new Pool({ connectionString: urlFor(name, role, rolePassword) }); + const pool = fixturePool(urlFor(name, role, rolePassword)); const failure = await exerciseStore(new PostgresStore(pool)).then( () => undefined, (error: unknown) => (error as { code?: string }).code From 04443607333349950d64f52c2c7766788c03b91e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:40:12 +0200 Subject: [PATCH 36/85] Schema check: valid, deterministic keys; Postgres-printed predicates; column grants; nullability Soul pass 6, F1-F4, F7 and F8. Keys (F1, F2): a key is met only by a unique index that is valid and ready, so the invalid index a failed CREATE UNIQUE INDEX CONCURRENTLY leaves behind (every sign-in failing with 42P10, or one consume taking two rows) is refused; and whose key columns compare under deterministic collations, so a case-insensitive collation cannot merge two providers' user ids into one row and move an identity between accounts. Predicates (F3, F7): the declared predicates are printed by Postgres itself, by indexing a temporary copy of each keyed table's declared columns in one transaction that is rolled back, and compared with the index's printed predicate as sets of top-level AND-ed conditions, split by nesting and never inside quoted text. Case and structure are kept, so ((a OR b) AND c) and (a OR (b AND c)) differ, as do 'Pending' and 'pending', and an extra condition narrows the predicate and is refused. Conditions in another order match. F4: lookup-only indexes are no longer required (REQUIRED_RELATIONS is now REQUIRED_TABLES; unique indexes are checked by shape in REQUIRED_KEYS), so a renamed attempt index or a dropped lookup index starts. Privileges are checked per column with has_column_privilege on every column the schema declares, which a table grant satisfies; a role granted each needed privilege column by column starts and runs every store operation, and one holding INSERT on only one audit column is refused. On (c): a unique index treats NULL keys as distinct unless it is NULLS NOT DISTINCT, so rows with a NULL attempt_id never collide and `attempt_id IS NOT NULL` is redundant; that condition may be absent from the attempt index's predicate, except on a NULLS NOT DISTINCT index (refused). A key declared without a predicate must have none, since ON CONFLICT cannot infer a partial arbiter. F8: a column the schema leaves nullable must be nullable; NOT NULL added to attempt_id is refused. CHECK constraints, triggers and row-level security stay out of scope, as the schema comment says. Every change is a live fixture: invalid identity key, invalid attempt index over duplicate attempts, case-insensitive key, narrowed and OR-widened predicates, NULLS NOT DISTINCT, NOT NULL added, partial column grants are refused; reordered and unredundant predicates, a renamed attempt index, dropped lookup indexes and column-by-column grants start. The fake-pool code-mapping tests are replaced by unit tests of the predicate comparison. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 212 ++++++++++++++++++++++++++++++++++++------ src/store/schema.ts | 42 ++++----- tests/store.test.ts | 192 +++++++++++++++++++++++++++----------- 3 files changed, 336 insertions(+), 110 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 4ba0ca7..20eccf0 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { Pool, type PoolClient, type QueryResultRow } from "pg"; import { type AppSlug, type HeimdallAuthAttemptStatus, type LinkedIdentityInput, type Provider } from "../contracts.js"; -import { CREATE_SCHEMA_SQL, REQUIRED_COLUMNS, REQUIRED_KEYS, REQUIRED_RELATIONS } from "./schema.js"; +import { CREATE_SCHEMA_SQL, REQUIRED_COLUMNS, REQUIRED_KEYS, REQUIRED_TABLES } from "./schema.js"; import { type CreateAccountInput, type CreateAuthAttemptInput, @@ -284,6 +284,85 @@ function mapAuthAttemptRow(row: AuthAttemptRow): StoredAuthAttempt { return attempt; } +/** + * The AND-ed conditions of a predicate as Postgres prints it, each without a + * pair of parentheses that encloses all of it. Postgres flattens a chain of + * ANDs into one list, so `((a) AND (b) AND (c))` yields `a`, `b`, `c`. An OR, + * or an AND nested under one, stays whole: `((a OR b) AND c)` yields `a OR b` + * and `c`, and `(a OR (b AND c))` yields itself. Quoted text is never split. + */ +export function predicateConditions(printed: string): string[] { + const unwrap = (text: string): string => { + let current = text.trim(); + while (current.startsWith("(") && closingParenthesis(current, 0) === current.length - 1) { + current = current.slice(1, -1).trim(); + } + return current; + }; + const whole = unwrap(printed); + if (whole === "") return []; + const parts: string[] = []; + let depth = 0; + let quote: string | undefined; + let start = 0; + for (let position = 0; position < whole.length; position += 1) { + const character = whole[position]!; + if (quote) { + if (character === quote) quote = undefined; + } else if (character === "'" || character === '"') { + quote = character; + } else if (character === "(") { + depth += 1; + } else if (character === ")") { + depth -= 1; + } else if (depth === 0 && whole.startsWith(" AND ", position)) { + parts.push(whole.slice(start, position)); + start = position + " AND ".length; + position = start - 1; + } + } + parts.push(whole.slice(start)); + return parts.map(unwrap); +} + +/** The index of the parenthesis closing the one at `open`, outside quoted text, or -1. */ +function closingParenthesis(text: string, open: number): number { + let depth = 0; + let quote: string | undefined; + for (let position = open; position < text.length; position += 1) { + const character = text[position]!; + if (quote) { + if (character === quote) quote = undefined; + } else if (character === "'" || character === '"') { + quote = character; + } else if (character === "(") { + depth += 1; + } else if (character === ")") { + depth -= 1; + if (depth === 0) return position; + } + } + return -1; +} + +/** + * Whether an index predicate, as Postgres prints it, keeps exactly the rows + * the declared one (printed the same way) does: the same set of AND-ed + * conditions, ignoring `key_column IS NOT NULL` unless the index treats NULLs + * as equal, since otherwise rows with a NULL key never collide anyway. + */ +export function samePredicate(declared: string, actual: string, keyColumns: string[], nullsNotDistinct: boolean): boolean { + // A key without a predicate must have none: ON CONFLICT cannot infer a + // partial index as its arbiter. + if (predicateConditions(declared).length === 0) return actual.trim() === ""; + const redundant = new Set(nullsNotDistinct ? [] : keyColumns.map((column) => `${column} IS NOT NULL`)); + const conditions = (printed: string) => + [...new Set(predicateConditions(printed).filter((condition) => !redundant.has(condition)))].sort(); + const left = conditions(declared); + const right = conditions(actual); + return left.length === right.length && left.every((condition, index) => condition === right[index]); +} + /** * What the store's statements do to each table, and so the privileges its * database role needs: it reads what it returns or looks up, inserts what it @@ -311,21 +390,43 @@ export class PostgresStore implements HeimdallStore { /** * Proves, before Heimdall serves, that the database can take its requests: - * every relation the schema makes, every column with its declared type, - * every unique key as the store relies on it, and every privilege in - * STORE_TABLE_PRIVILEGES. A missing or altered piece fails with code - * SCHEMA_MISSING and a missing privilege with SCHEMA_PRIVILEGES; neither - * error names the piece. + * every table the schema makes; every column with its declared type, and + * nullable where the schema leaves it nullable; every unique key; and every + * privilege in STORE_TABLE_PRIVILEGES, held on the table or on each of its + * columns. A missing or altered piece fails with code SCHEMA_MISSING and a + * missing privilege with SCHEMA_PRIVILEGES; neither error names the piece. + * + * A key is met by a unique index on its table that is valid and ready (a + * failed CREATE INDEX CONCURRENTLY leaves one that is neither), immediate + * (ON CONFLICT refuses a deferrable arbiter), has exactly the key's columns + * as its key columns (INCLUDE columns aside), compares them under + * deterministic collations (a case-insensitive one would merge two + * providers' user ids into one row), and has the key's predicate. A + * predicate matches when both, as Postgres prints them, are the same set of + * AND-ed conditions; a `key_column IS NOT NULL` condition may be absent on + * either side, because a unique index never makes NULL keys collide unless + * it is NULLS NOT DISTINCT. + * + * Postgres prints the declared predicates itself: they are indexed on a + * temporary copy of the table in a transaction that is rolled back. */ async checkSchema(): Promise { + const declaredPredicates = await this.printedDeclaredPredicates(); const privileges = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => wanted.map((privilege) => ({ table, privilege })) ); - const result = await this.pool.query<{ kind: "missing" | "privilege" }>( + const result = await this.pool.query<{ + kind: "missing" | "privilege" | "candidate"; + key: number | null; + predicate: string | null; + nulls_not_distinct: boolean | null; + }>( ` - SELECT 'missing' AS kind FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL + SELECT 'missing' AS kind, NULL::int AS key, NULL::text AS predicate, NULL::boolean AS nulls_not_distinct + FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL UNION ALL - SELECT 'missing' FROM unnest($2::text[], $3::text[], $4::text[]) AS required(table_name, column_name, type_name) + SELECT 'missing', NULL, NULL, NULL + FROM unnest($2::text[], $3::text[], $4::text[], $5::boolean[]) AS required(table_name, column_name, type_name, nullable) WHERE NOT EXISTS ( SELECT 1 FROM pg_attribute a WHERE a.attrelid = to_regclass(required.table_name) @@ -333,50 +434,99 @@ export class PostgresStore implements HeimdallStore { AND a.attnum > 0 AND NOT a.attisdropped AND a.atttypid = to_regtype(required.type_name) + AND (NOT required.nullable OR NOT a.attnotnull) ) UNION ALL - SELECT 'missing' FROM unnest($5::text[], $6::text[], $7::text[]) AS required(table_name, columns, predicate) - WHERE NOT EXISTS ( - SELECT 1 FROM pg_index i - WHERE i.indrelid = to_regclass(required.table_name) - AND i.indisunique - AND i.indimmediate - AND coalesce(regexp_replace(lower(pg_get_expr(i.indpred, i.indrelid)), '[()[:space:]]', '', 'g'), '') = required.predicate - AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) - AND ( - SELECT string_agg(a.attname, ',' ORDER BY a.attname) - FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) - JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum - WHERE key.position <= i.indnkeyatts - ) = required.columns - ) + SELECT 'candidate', required.key::int, pg_get_expr(i.indpred, i.indrelid), + coalesce((to_jsonb(i) ->> 'indnullsnotdistinct')::boolean, false) + FROM unnest($6::text[], $7::text[]) WITH ORDINALITY AS required(table_name, columns, key) + JOIN pg_index i ON i.indrelid = to_regclass(required.table_name) + WHERE i.indisunique + AND i.indisvalid + AND i.indisready + AND i.indimmediate + AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) + AND ( + SELECT string_agg(a.attname, ',' ORDER BY a.attname) + FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum + WHERE key.position <= i.indnkeyatts + ) = required.columns + AND NOT EXISTS ( + SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS collation(oid, position) + JOIN pg_collation c ON c.oid = collation.oid + WHERE collation.position <= i.indnkeyatts AND NOT c.collisdeterministic + ) UNION ALL - SELECT 'privilege' FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) - WHERE to_regclass(required.table_name) IS NOT NULL - AND NOT has_table_privilege(to_regclass(required.table_name), required.privilege) + SELECT 'privilege', NULL, NULL, NULL + FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) + WHERE EXISTS ( + SELECT 1 FROM unnest($2::text[], $3::text[]) AS used(table_name, column_name) + JOIN pg_attribute a ON a.attrelid = to_regclass(used.table_name) AND a.attname = used.column_name AND NOT a.attisdropped + WHERE used.table_name = required.table_name + AND NOT has_column_privilege(a.attrelid, a.attnum, required.privilege) + ) `, [ - REQUIRED_RELATIONS, + REQUIRED_TABLES, REQUIRED_COLUMNS.map((required) => required.table), REQUIRED_COLUMNS.map((required) => required.column), REQUIRED_COLUMNS.map((required) => required.type), + REQUIRED_COLUMNS.map((required) => required.nullable), REQUIRED_KEYS.map((required) => required.table), REQUIRED_KEYS.map((required) => required.columns), - REQUIRED_KEYS.map((required) => required.predicate), privileges.map((required) => required.table), privileges.map((required) => required.privilege), ] ); - if (result.rows.some((row) => row.kind === "missing")) { + const keyMet = REQUIRED_KEYS.map((required, index) => + result.rows.some( + (row) => + row.kind === "candidate" && + row.key === index + 1 && + samePredicate(declaredPredicates[index]!, row.predicate ?? "", required.columns.split(","), row.nulls_not_distinct === true) + ) + ); + if (result.rows.some((row) => row.kind === "missing") || keyMet.includes(false)) { throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); } - if (result.rows.length > 0) { + if (result.rows.some((row) => row.kind === "privilege")) { throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { code: "SCHEMA_PRIVILEGES", }); } } + /** + * Each REQUIRED_KEYS predicate as Postgres prints it back (empty for none). + * One simple-protocol round trip on one connection: a transaction that + * builds a temporary copy of each keyed table's declared columns, indexes + * it with the declared predicate, reads the printed form and rolls back. + * The names and SQL are the schema's own constants, never input. + */ + private async printedDeclaredPredicates(): Promise { + const partial = REQUIRED_KEYS.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); + if (partial.length === 0) return REQUIRED_KEYS.map(() => ""); + const statements = partial.flatMap(({ table, columns, predicate, index }) => { + const probe = `heimdall_key_probe_${index}`; + const declared = REQUIRED_COLUMNS.filter((column) => column.table === table) + .map((column) => `${column.column} ${column.type}`) + .join(", "); + return [ + `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, + `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, + ]; + }); + const select = partial + .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) + .join(" UNION ALL "); + const results = (await this.pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as Array<{ + rows: Array<{ key: number; predicate: string }>; + }>; + const printed = results[results.length - 2]!.rows; + return REQUIRED_KEYS.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); + } + async close(): Promise { await this.pool.end(); } diff --git a/src/store/schema.ts b/src/store/schema.ts index 91db7fc..f3588c0 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -180,30 +180,25 @@ function keyColumns(line: string, keyword: "PRIMARY KEY" | "UNIQUE"): string | u return match ? columnList(match[1]!) : undefined; } -/** - * An index predicate in the form the schema check compares: lower case, with - * no parentheses or white space. Postgres stores `a IS NOT NULL AND b IS NULL` - * back as `((a IS NOT NULL) AND (b IS NULL))`; both reduce to the same text. - */ -export function comparablePredicate(predicate: string): string { - return predicate.toLowerCase().replace(/[()\s]/g, ""); -} - /* * What every start must find before it serves, read from the SQL above so the * two cannot drift. A start that applies the schema checks as well, because - * CREATE ... IF NOT EXISTS leaves a damaged table as it is. + * CREATE ... IF NOT EXISTS leaves a damaged table as it is. Indexes that only + * speed lookups are not required; the unique ones are, by shape, in + * REQUIRED_KEYS. + * + * Out of scope: CHECK constraints, triggers and row-level security. Each can + * make a write fail on a database that passes this check. */ -/** Every table and index the schema creates. */ -export const REQUIRED_RELATIONS = [ - ...schemaText.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), -].map((match) => match[1]!); +/** Every table the schema creates. */ +export const REQUIRED_TABLES = createdTables.map(({ table }) => table); /** * Every column a created table declares and every column the schema adds to - * an existing table, each with its declared type: the store reads and writes - * the values as that type, and a column retyped under it fails its queries. + * an existing table, each with its declared type, and whether it is nullable: + * the store reads and writes values as that type, and leaves a nullable + * column null, so a column retyped or made NOT NULL under it fails its writes. */ export const REQUIRED_COLUMNS = [ ...createdTables.flatMap(({ table, lines }) => @@ -211,24 +206,23 @@ export const REQUIRED_COLUMNS = [ .filter((line) => !/^(?:PRIMARY KEY|UNIQUE)\b/.test(line)) .map((line) => { const [, column, type] = /^(\w+) (\w+)/.exec(line)!; - return { table, column: column!, type: type! }; + return { table, column: column!, type: type!, nullable: !/\b(?:NOT NULL|PRIMARY KEY)\b/.test(line) }; }) ), - ...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+) (\w+)/gm)].map((match) => ({ + ...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+) (\w+)([^;]*);/gm)].map((match) => ({ table: match[1]!, column: match[2]!, type: match[3]!, + nullable: !/\b(?:NOT NULL|PRIMARY KEY)\b/.test(match[4]!), })), ]; /** * Every unique key the schema makes: the primary keys and unique constraints * of the created tables, and each unique index it creates, with that index's - * predicate. The store relies on each as an immediate unique index on its - * table whose key columns are exactly these, with exactly this predicate: ON - * CONFLICT refuses a deferrable arbiter and cannot infer a partial or wider - * one, and the partial attempt index is what keeps one unconsumed completion - * per attempt (R21.1). Any index with that shape serves, whatever its name. + * predicate as the schema writes it (empty for none). Any index of the right + * shape serves, whatever its name; PostgresStore.checkSchema says what the + * right shape is. */ export const REQUIRED_KEYS = [ ...createdTables.flatMap(({ table, lines }) => @@ -241,6 +235,6 @@ export const REQUIRED_KEYS = [ ...[...schemaText.matchAll(/^CREATE UNIQUE INDEX IF NOT EXISTS \w+\s+ON (\w+)\(([^)]*)\)(?:\s+WHERE ([^;]+))?;/gm)].map((match) => ({ table: match[1]!, columns: columnList(match[2]!), - predicate: comparablePredicate(match[3] ?? ""), + predicate: (match[3] ?? "").replace(/\s+/g, " ").trim(), })), ]; diff --git a/tests/store.test.ts b/tests/store.test.ts index 25500c7..22a694b 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -4,8 +4,14 @@ import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { newDb } from "pg-mem"; import { type HeimdallConfig } from "../src/config.js"; import { createStore } from "../src/store/index.js"; -import { createPostgresStore, PostgresStore, STORE_TABLE_PRIVILEGES } from "../src/store/postgres.js"; -import { comparablePredicate, REQUIRED_COLUMNS, REQUIRED_KEYS, REQUIRED_RELATIONS } from "../src/store/schema.js"; +import { + createPostgresStore, + PostgresStore, + predicateConditions, + samePredicate, + STORE_TABLE_PRIVILEGES, +} from "../src/store/postgres.js"; +import { REQUIRED_COLUMNS, REQUIRED_KEYS, REQUIRED_TABLES } from "../src/store/schema.js"; describe("PostgresStore", () => { it("round-trips core auth records through postgres", async () => { @@ -216,19 +222,27 @@ describe("createPostgresStore", () => { }); describe("PostgresStore.checkSchema", () => { - it("derives every relation, typed column and unique key from the schema", () => { - expect(REQUIRED_RELATIONS).toEqual( - expect.arrayContaining(["accounts", "audit_events", "auth_completions_attempt_unconsumed_unique_idx", "audit_events_lookup_idx"]) - ); - expect(REQUIRED_RELATIONS).toHaveLength(17); + it("derives every table, typed column and unique key from the schema", () => { + expect(REQUIRED_TABLES).toEqual([ + "accounts", + "linked_identities", + "sessions", + "auth_attempts", + "private_command_receipts", + "auth_completions", + "capability_grants", + "entitlement_snapshots", + "audit_events", + ]); expect(REQUIRED_COLUMNS).toHaveLength(85); expect(REQUIRED_COLUMNS).toEqual( expect.arrayContaining([ - { table: "auth_completions", column: "code", type: "TEXT" }, - { table: "sessions", column: "access_revision", type: "INTEGER" }, - { table: "linked_identities", column: "profile_json", type: "JSONB" }, - { table: "entitlement_snapshots", column: "is_allowed", type: "BOOLEAN" }, - { table: "auth_completions", column: "attempt_id", type: "TEXT" }, + { table: "auth_completions", column: "code", type: "TEXT", nullable: false }, + { table: "sessions", column: "access_revision", type: "INTEGER", nullable: false }, + { table: "linked_identities", column: "profile_json", type: "JSONB", nullable: false }, + { table: "linked_identities", column: "username", type: "TEXT", nullable: true }, + { table: "entitlement_snapshots", column: "is_allowed", type: "BOOLEAN", nullable: false }, + { table: "auth_completions", column: "attempt_id", type: "TEXT", nullable: true }, ]) ); expect(REQUIRED_KEYS).toHaveLength(12); @@ -237,38 +251,43 @@ describe("PostgresStore.checkSchema", () => { { table: "linked_identities", columns: "provider,provider_user_id", predicate: "" }, { table: "private_command_receipts", columns: "app_slug,idempotency_key", predicate: "" }, { table: "entitlement_snapshots", columns: "account_id,provider,scope", predicate: "" }, - { - table: "auth_completions", - columns: "app_slug,attempt_id", - predicate: comparablePredicate("attempt_id IS NOT NULL AND consumed_at IS NULL"), - }, + { table: "auth_completions", columns: "app_slug,attempt_id", predicate: "attempt_id IS NOT NULL AND consumed_at IS NULL" }, ]) ); }); - it("compares a predicate in the form Postgres stores it back", () => { - expect(comparablePredicate("((attempt_id IS NOT NULL) AND (consumed_at IS NULL))")).toBe( - comparablePredicate("attempt_id IS NOT NULL AND consumed_at IS NULL") - ); + // Soul pass 6: stripping parentheses and case merged predicates of + // different meaning. These are Postgres's own printed forms. + it("splits a printed predicate only at its top-level ANDs", () => { + expect(predicateConditions("((a OR b) AND c)")).toEqual(["a OR b", "c"]); + expect(predicateConditions("(a OR (b AND c))")).toEqual(["a OR (b AND c)"]); + expect(predicateConditions("((attempt_id IS NOT NULL) AND (consumed_at IS NULL))")).toEqual([ + "attempt_id IS NOT NULL", + "consumed_at IS NULL", + ]); + expect(predicateConditions("(s = 'x AND (y'::text)")).toEqual(["s = 'x AND (y'::text"]); + expect(predicateConditions("")).toEqual([]); }); - function storeAnswering(kinds: string[]) { - return new PostgresStore({ - query: (async () => ({ rows: kinds.map((kind) => ({ kind })) })) as never, - end: async () => undefined, - }); - } - - it("refuses with SCHEMA_MISSING when anything is missing, whatever else is", async () => { - await expect(storeAnswering(["privilege", "missing"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" }); + it("matches predicates by their conditions, never by case or by dropping structure", () => { + const keys = ["app_slug", "attempt_id"]; + expect(samePredicate("((a OR b) AND c)", "(a OR (b AND c))", keys, false)).toBe(false); + expect(samePredicate("(s = 'Pending'::text)", "(s = 'pending'::text)", keys, false)).toBe(false); + expect(samePredicate("((a) AND (b))", "((b) AND (a))", keys, false)).toBe(true); + expect(samePredicate("((a) AND (b))", "((a) AND (b) AND (c))", keys, false)).toBe(false); }); - it("refuses with SCHEMA_PRIVILEGES when only a privilege is missing", async () => { - await expect(storeAnswering(["privilege"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_PRIVILEGES" }); + it("lets a key column's IS NOT NULL be absent only where NULL keys cannot collide", () => { + const keys = ["app_slug", "attempt_id"]; + const declared = "((attempt_id IS NOT NULL) AND (consumed_at IS NULL))"; + expect(samePredicate(declared, "(consumed_at IS NULL)", keys, false)).toBe(true); + expect(samePredicate(declared, "(consumed_at IS NULL)", keys, true)).toBe(false); + expect(samePredicate(declared, "((consumed_at IS NULL) AND (other IS NOT NULL))", keys, false)).toBe(false); }); - it("passes when nothing is missing", async () => { - await expect(storeAnswering([]).checkSchema()).resolves.toBeUndefined(); + it("requires a key without a predicate to have none", () => { + expect(samePredicate("", "", ["id"], false)).toBe(true); + expect(samePredicate("", "(id IS NOT NULL)", ["id"], false)).toBe(false); }); }); @@ -470,9 +489,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD it("refuses each privilege taken away, and the store needs each one", async () => { const pairs = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => wanted.map((privilege) => ({ table, privilege }))); - expect(Object.keys(STORE_TABLE_PRIVILEGES).sort()).toEqual( - REQUIRED_RELATIONS.filter((relation) => REQUIRED_COLUMNS.some((column) => column.table === relation)).sort() - ); + expect(Object.keys(STORE_TABLE_PRIVILEGES).sort()).toEqual([...REQUIRED_TABLES].sort()); for (const { table, privilege } of pairs) { const { name, owner } = await appliedDatabase(); await grant(owner, { ...STORE_TABLE_PRIVILEGES, [table]: STORE_TABLE_PRIVILEGES[table]!.filter((p) => p !== privilege) }); @@ -507,41 +524,106 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD expect(await start(urlFor(name), true)).toBe(refusal("SCHEMA_MISSING")); }); + it("serves every store operation with each privilege granted column by column", async () => { + const { name, owner } = await appliedDatabase(); + await owner.query(`GRANT USAGE ON SCHEMA public TO ${role}`); + for (const [table, wanted] of Object.entries(STORE_TABLE_PRIVILEGES)) { + const columns = REQUIRED_COLUMNS.filter((column) => column.table === table).map((column) => column.column).join(", "); + for (const privilege of wanted) await owner.query(`GRANT ${privilege} (${columns}) ON ${table} TO ${role}`); + } + + expect(await start(urlFor(name, role, rolePassword), false)).toBe("started"); + const pool = fixturePool(urlFor(name, role, rolePassword)); + cleanup.push(async () => { + await pool.end(); + }); + await exerciseStore(new PostgresStore(pool)); + }); + + it("refuses a privilege held on only some of a table's columns", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, { ...STORE_TABLE_PRIVILEGES, audit_events: [] }); + await owner.query(`GRANT INSERT (id) ON audit_events TO ${role}`); + + expect(await start(urlFor(name, role, rolePassword), false)).toBe(refusal("SCHEMA_PRIVILEGES")); + }); + // Each damage below leaves a database whose shape still resembles the - // schema but on which a store write fails or R21.1 no longer holds. + // schema but on which a store write fails or R21.1 no longer holds. A step + // starting with "!" is expected to fail and leave its mark (a failed + // CREATE INDEX CONCURRENTLY leaves an invalid index behind). + const seedDuplicates = [ + "INSERT INTO accounts VALUES ('acc', now(), now(), null, null)", + "INSERT INTO sessions VALUES ('s', 'acc', 'ghostlight', now(), now(), now() + interval '1 hour', '{}', 1)", + ]; it.each([ ["a key made DEFERRABLE (ON CONFLICT refuses it as an arbiter)", - "ALTER TABLE private_command_receipts DROP CONSTRAINT private_command_receipts_pkey; ALTER TABLE private_command_receipts ADD PRIMARY KEY (app_slug, idempotency_key) DEFERRABLE INITIALLY IMMEDIATE"], + ["ALTER TABLE private_command_receipts DROP CONSTRAINT private_command_receipts_pkey", "ALTER TABLE private_command_receipts ADD PRIMARY KEY (app_slug, idempotency_key) DEFERRABLE INITIALLY IMMEDIATE"]], ["a key narrowed to one column, the other only INCLUDEd", - "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_include ON linked_identities(provider) INCLUDE (provider_user_id)"], + ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE UNIQUE INDEX li_include ON linked_identities(provider) INCLUDE (provider_user_id)"]], ["a key widened by an expression column", - "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_expr ON linked_identities(provider, provider_user_id, lower(username))"], + ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE UNIQUE INDEX li_expr ON linked_identities(provider, provider_user_id, lower(username))"]], ["a key replaced by a partial one", - "ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key; CREATE UNIQUE INDEX es_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed"], + ["ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key", "CREATE UNIQUE INDEX es_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed"]], ["a key replaced by a plain index on the same columns", - "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE INDEX li_plain ON linked_identities(provider, provider_user_id)"], + ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE INDEX li_plain ON linked_identities(provider, provider_user_id)"]], ["a key replaced by one on other columns of the same count", - "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_other ON linked_identities(provider, username)"], + ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE UNIQUE INDEX li_other ON linked_identities(provider, username)"]], + ["a key left only as an invalid index by a failed concurrent build", + [...seedDuplicates, + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "INSERT INTO linked_identities (id, account_id, provider, provider_user_id, created_at, updated_at) VALUES ('l1', 'acc', 'youtube', 'dup', now(), now()), ('l2', 'acc', 'youtube', 'dup', now(), now())", + "!CREATE UNIQUE INDEX CONCURRENTLY li_ccnew ON linked_identities(provider, provider_user_id)", + "DELETE FROM linked_identities WHERE id = 'l2'"]], + ["the attempt index left invalid over duplicate attempts by a failed concurrent build", + [...seedDuplicates, + "DROP INDEX auth_completions_attempt_unconsumed_unique_idx", + "INSERT INTO auth_completions (code, attempt_id, app_slug, provider, mode, account_id, session_id, return_to, payload_json, created_at, expires_at) VALUES ('c1', 'att', 'ghostlight', 'discord', 'sign_in', 'acc', 's', 'https://x/', '{}', now(), now() + interval '1 hour'), ('c2', 'att', 'ghostlight', 'discord', 'sign_in', 'acc', 's', 'https://x/', '{}', now(), now() + interval '1 hour')", + "!CREATE UNIQUE INDEX CONCURRENTLY auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE attempt_id IS NOT NULL AND consumed_at IS NULL"]], + ["a key compared under a case-insensitive collation (two users' ids become one row)", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "CREATE UNIQUE INDEX li_ci ON linked_identities(provider, provider_user_id COLLATE heimdall_ci)"]], ["the attempt index replaced by a plain one of the same name", - "DROP INDEX auth_completions_attempt_unconsumed_unique_idx; CREATE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id)"], + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id)"]], ["the attempt index made unique on every row (no predicate)", - "DROP INDEX auth_completions_attempt_unconsumed_unique_idx; CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id)"], + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id)"]], ["the attempt index given another predicate", - "DROP INDEX auth_completions_attempt_unconsumed_unique_idx; CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE attempt_id IS NOT NULL"], - ["a column retyped", "ALTER TABLE sessions ALTER COLUMN access_revision TYPE TEXT"], - ["a created table's column dropped", "ALTER TABLE auth_completions DROP COLUMN mode"], - ])("refuses %s", async (_damage, sql) => { + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE attempt_id IS NOT NULL"]], + ["the attempt index narrowed by an extra condition", + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE attempt_id IS NOT NULL AND consumed_at IS NULL AND app_slug <> 'ghostlight'"]], + ["the attempt index with a condition ORed into its predicate", + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE (attempt_id IS NOT NULL OR app_slug = 'x') AND consumed_at IS NULL"]], + ["the attempt index made NULLS NOT DISTINCT without excluding NULL attempts", + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) NULLS NOT DISTINCT WHERE consumed_at IS NULL"]], + ["a column retyped", ["ALTER TABLE sessions ALTER COLUMN access_revision TYPE TEXT"]], + ["a created table's column dropped", ["ALTER TABLE auth_completions DROP COLUMN mode"]], + ["NOT NULL added to a column the store leaves null", ["ALTER TABLE auth_completions ALTER COLUMN attempt_id SET NOT NULL"]], + ])("refuses %s", async (_damage, steps) => { const { name, owner } = await appliedDatabase(); - await owner.query(sql); + for (const step of steps) { + if (step.startsWith("!")) await expect(owner.query(step.slice(1))).rejects.toBeDefined(); + else await owner.query(step); + } expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); }); - it("accepts a key index that carries extra INCLUDE columns", async () => { + // Each change below leaves a database the store serves exactly as before. + it.each([ + ["a key index that carries extra INCLUDE columns", + ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE UNIQUE INDEX li_covering ON linked_identities(provider, provider_user_id) INCLUDE (username)"]], + ["the attempt index under another name", ["ALTER INDEX auth_completions_attempt_unconsumed_unique_idx RENAME TO attempt_idx_renamed"]], + ["the attempt index's conditions in the other order", + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX attempt_reordered ON auth_completions(app_slug, attempt_id) WHERE consumed_at IS NULL AND attempt_id IS NOT NULL"]], + // NULL attempt ids never collide in a unique index that treats NULLs as + // distinct (the default), so dropping that condition keeps the same rows unique. + ["the attempt index without its redundant attempt_id IS NOT NULL", + ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX attempt_unconsumed ON auth_completions(app_slug, attempt_id) WHERE consumed_at IS NULL"]], + ["a lookup index dropped", ["DROP INDEX audit_events_lookup_idx", "DROP INDEX auth_completions_lookup_idx"]], + ])("accepts %s", async (_change, steps) => { const { name, owner } = await appliedDatabase(); - await owner.query( - "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key; CREATE UNIQUE INDEX li_covering ON linked_identities(provider, provider_user_id) INCLUDE (username)" - ); + for (const step of steps) await owner.query(step); expect(await start(urlFor(name), false)).toBe("started"); }); From d2ac9696c7f4b2f92fbc58c23a5d7cb2750c983a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:40:13 +0200 Subject: [PATCH 37/85] Recipe test: combined runs from accepted values, config literals as probes, fresh import Soul pass 6, F5 and F6. The all-at-once run set every name to one probe value and died in loadConfig's first lines, so reads gated on a specific value (Z4) or on two values late in the function (Z5) were missed, and a copy of process.env taken at module load (Z6) was never the recorder. Each name now gets each probe value alone, and the probe values include every string literal in config.ts; the combined runs set every name at once to values loadConfig accepted for it alone. config.ts is imported afresh while the recorder is process.env, so a module-scope capture records too. The remaining limit is stated in the test. failureOf in the app tests now prints the error with util.inspect as well, so a non-enumerable cause carrying the body (Z3) shows. Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 4 +- tests/recipe-secrets.test.ts | 132 +++++++++++++++++++++-------------- 2 files changed, 82 insertions(+), 54 deletions(-) diff --git a/tests/app.test.ts b/tests/app.test.ts index 044191a..7f9a72c 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1,3 +1,4 @@ +import { inspect } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { type FastifyInstance } from "fastify"; import { buildApp, errorIdentity } from "../src/app.js"; @@ -1699,7 +1700,8 @@ describe("a provider answer Heimdall cannot use", () => { expect(error).toBeInstanceOf(Error); return { error: error as Error & { code?: unknown; status?: unknown }, - printed: `${String(error)} ${(error as Error).stack ?? ""} ${JSON.stringify(error)}`, + // inspect shows what the others miss, such as a non-enumerable cause. + printed: `${String(error)} ${(error as Error).stack ?? ""} ${JSON.stringify(error)} ${inspect(error, { depth: 5, showHidden: true })}`, }; } diff --git a/tests/recipe-secrets.test.ts b/tests/recipe-secrets.test.ts index 31717c3..d7f69ec 100644 --- a/tests/recipe-secrets.test.ts +++ b/tests/recipe-secrets.test.ts @@ -1,8 +1,7 @@ import { readFileSync } from "node:fs"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { appProfiles } from "../src/app-profiles.js"; import { appSlugs, providers } from "../src/contracts.js"; -import { loadConfig } from "../src/config.js"; import { providerCatalog } from "../src/providers.js"; // The Idunn recipe decides which environment names a binding may set, and @@ -31,80 +30,107 @@ const everyDeclaredName = [...recipe.replace(/\r\n/g, "\n").matchAll(/^\w+_envir // loadConfig runs against an environment that records every name it looks // up. readSecretInput looks up `NAME_FILE` for each secret, so every `_FILE` // lookup names one, however the call is spelled and whatever list computes -// the name. The same recorder stands in for process.env during the call, so a -// read that bypasses the env argument is seen too. +// the name. The recorder also stands in for process.env, both while config.ts +// is imported (freshly, so a copy of process.env taken at module load is the +// recorder too) and during every call. // // A read made from a copy of the environment ({ ...env }, Object.entries) // cannot be seen, so copying or enumerating it is recorded and fails below. // -// A secret read only on a branch that some value opens is reached by running -// loadConfig over many environments: every secret found so far set in -// plaintext, then each name read so far set to each probe value in turn, then -// every such name set to one probe value at once, until no run reads a new -// name. A run that throws still counts the reads it made first. -const probeValues = ["1", "true", "/probe/path", "127.0.0.1:4100", "https://probe.test"]; +// A secret read only on a branch some value opens is reached by running +// loadConfig over many environments, until no run reads a new name: every +// secret found so far set in plaintext; then each name read so far set to +// each probe value (a few shapes, plus every string literal in config.ts); +// then every name at once, each set to a value loadConfig accepted for it +// alone. A run that throws still counts the reads it made first. +// +// Known limit: a read gated on a value that is none of the probe values and +// not accepted alone in combination with the others stays unseen. +const configSource = readFileSync(new URL("../src/config.ts", import.meta.url), "utf8"); +const probeValues = [ + ...new Set(["1", "true", "/probe/path", "127.0.0.1:4100", "https://probe.test", ...[...configSource.matchAll(/"([^"\\\r\n]+)"/g)].map((m) => m[1]!)]), +]; -function readsOf(values: Readonly>): { names: Set; enumerated: boolean } { - const names = new Set(); - let enumerated = false; - const record = (name: string | symbol) => { - if (typeof name === "string") names.add(name); - }; - const env = new Proxy({} as NodeJS.ProcessEnv, { - get(_target, name) { - record(name); - return typeof name === "string" ? values[name] : undefined; - }, - has(_target, name) { - record(name); - return typeof name === "string" && name in values; - }, - ownKeys() { - enumerated = true; - return []; - }, - getOwnPropertyDescriptor() { - enumerated = true; - return undefined; - }, - }); +const recorder = { values: {} as Record, names: new Set(), enumerated: false }; +const recordingEnv = new Proxy({} as NodeJS.ProcessEnv, { + get(_target, name) { + if (typeof name !== "string") return undefined; + recorder.names.add(name); + return recorder.values[name]; + }, + has(_target, name) { + if (typeof name !== "string") return false; + recorder.names.add(name); + return name in recorder.values; + }, + ownKeys() { + recorder.enumerated = true; + return []; + }, + getOwnPropertyDescriptor() { + recorder.enumerated = true; + return undefined; + }, +}); + +async function withRecordingProcessEnv(run: () => T | Promise): Promise { const processEnv = process.env; - process.env = env; + process.env = recordingEnv; try { - loadConfig(env, []); - } catch { - // The reads before the refusal still count. + return await run(); } finally { process.env = processEnv; } - return { names, enumerated }; } -function exploreLoadConfig(): { names: Set; enumerated: boolean } { +vi.resetModules(); +const { loadConfig } = await withRecordingProcessEnv(() => import("../src/config.js")); + +/** The names one loadConfig run reads under `values`, and whether it accepted them. */ +async function readsOf(values: Readonly>): Promise<{ names: Set; accepted: boolean }> { + recorder.values = { ...values }; + recorder.names = new Set(); + let accepted = true; + await withRecordingProcessEnv(() => { + try { + loadConfig(recordingEnv, []); + } catch { + accepted = false; // the reads before the refusal still count + } + }); + return { names: recorder.names, accepted }; +} + +async function exploreLoadConfig(): Promise<{ names: Set; enumerated: boolean }> { const names = new Set(); - let enumerated = false; + recorder.enumerated = false; const secretsOf = () => [...names].filter((name) => name.endsWith("_FILE")).map((name) => name.slice(0, -"_FILE".length)); for (let grew = true; grew; ) { const before = names.size; const secrets = Object.fromEntries(secretsOf().map((name) => [name, "probe-secret"])); const settable = [...names].filter((name) => !name.endsWith("_FILE") && !(name in secrets)); - const runs: Array> = [ - {}, - secrets, - ...settable.flatMap((name) => probeValues.map((value) => ({ ...secrets, [name]: value }))), - ...probeValues.map((value) => ({ ...secrets, ...Object.fromEntries(settable.map((name) => [name, value])) })), - ]; - for (const values of runs) { - const run = readsOf(values); - run.names.forEach((name) => names.add(name)); - enumerated ||= run.enumerated; + const accepted = new Map(); + const keep = (run: { names: Set }) => run.names.forEach((name) => names.add(name)); + keep(await readsOf({})); + keep(await readsOf(secrets)); + for (const name of settable) { + for (const value of probeValues) { + const run = await readsOf({ ...secrets, [name]: value }); + keep(run); + if (run.accepted) accepted.set(name, [...(accepted.get(name) ?? []), value]); + } + } + const widest = Math.max(0, ...[...accepted.values()].map((values) => values.length)); + for (let round = 0; round < widest; round += 1) { + const together = Object.fromEntries([...accepted].map(([name, values]) => [name, values[round % values.length]!])); + keep(await readsOf({ ...secrets, ...together })); } grew = names.size > before; } - return { names, enumerated }; + return { names, enumerated: recorder.enumerated }; } -const explored = exploreLoadConfig(); +const explored = await exploreLoadConfig(); /** Secrets Heimdall reads through readSecretInput, by their plaintext name. */ const secretNames = [...explored.names] From eff5098c2324c267ddeb4cf3db12b539b5ac6814 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:58:29 +0200 Subject: [PATCH 38/85] Schema check: the collation alias is not a reserved word `collation` is reserved in Postgres, so the key check's query failed with 42601 on every database and every start refused. The live block showed it. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 20eccf0..6f10326 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -453,9 +453,9 @@ export class PostgresStore implements HeimdallStore { WHERE key.position <= i.indnkeyatts ) = required.columns AND NOT EXISTS ( - SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS collation(oid, position) - JOIN pg_collation c ON c.oid = collation.oid - WHERE collation.position <= i.indnkeyatts AND NOT c.collisdeterministic + SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS key_collation(oid, position) + JOIN pg_collation c ON c.oid = key_collation.oid + WHERE key_collation.position <= i.indnkeyatts AND NOT c.collisdeterministic ) UNION ALL SELECT 'privilege', NULL, NULL, NULL From 97632b7cfa948f289b4ed72c6ed8a548b9e9323b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 21:11:18 +0200 Subject: [PATCH 39/85] Print the declared predicates in a module function, not a store method The store's coverage test lists every method on PostgresStore as an operation the workload must call, and the private predicate printer counted as one. It is a helper of checkSchema, not a store operation, so it becomes a function of the pool. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 62 +++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 6f10326..fc9e687 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -363,6 +363,36 @@ export function samePredicate(declared: string, actual: string, keyColumns: stri return left.length === right.length && left.every((condition, index) => condition === right[index]); } +/** + * Each REQUIRED_KEYS predicate as Postgres prints it back (empty for none). + * One simple-protocol round trip on one connection: a transaction that + * builds a temporary copy of each keyed table's declared columns, indexes + * it with the declared predicate, reads the printed form and rolls back. + * The names and SQL are the schema's own constants, never input. + */ +async function printedDeclaredPredicates(pool: Pick): Promise { + const partial = REQUIRED_KEYS.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); + if (partial.length === 0) return REQUIRED_KEYS.map(() => ""); + const statements = partial.flatMap(({ table, columns, predicate, index }) => { + const probe = `heimdall_key_probe_${index}`; + const declared = REQUIRED_COLUMNS.filter((column) => column.table === table) + .map((column) => `${column.column} ${column.type}`) + .join(", "); + return [ + `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, + `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, + ]; + }); + const select = partial + .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) + .join(" UNION ALL "); + const results = (await pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as Array<{ + rows: Array<{ key: number; predicate: string }>; + }>; + const printed = results[results.length - 2]!.rows; + return REQUIRED_KEYS.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); +} + /** * What the store's statements do to each table, and so the privileges its * database role needs: it reads what it returns or looks up, inserts what it @@ -411,7 +441,7 @@ export class PostgresStore implements HeimdallStore { * temporary copy of the table in a transaction that is rolled back. */ async checkSchema(): Promise { - const declaredPredicates = await this.printedDeclaredPredicates(); + const declaredPredicates = await printedDeclaredPredicates(this.pool); const privileges = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => wanted.map((privilege) => ({ table, privilege })) ); @@ -497,36 +527,6 @@ export class PostgresStore implements HeimdallStore { } } - /** - * Each REQUIRED_KEYS predicate as Postgres prints it back (empty for none). - * One simple-protocol round trip on one connection: a transaction that - * builds a temporary copy of each keyed table's declared columns, indexes - * it with the declared predicate, reads the printed form and rolls back. - * The names and SQL are the schema's own constants, never input. - */ - private async printedDeclaredPredicates(): Promise { - const partial = REQUIRED_KEYS.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); - if (partial.length === 0) return REQUIRED_KEYS.map(() => ""); - const statements = partial.flatMap(({ table, columns, predicate, index }) => { - const probe = `heimdall_key_probe_${index}`; - const declared = REQUIRED_COLUMNS.filter((column) => column.table === table) - .map((column) => `${column.column} ${column.type}`) - .join(", "); - return [ - `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, - `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, - ]; - }); - const select = partial - .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) - .join(" UNION ALL "); - const results = (await this.pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as Array<{ - rows: Array<{ key: number; predicate: string }>; - }>; - const printed = results[results.length - 2]!.rows; - return REQUIRED_KEYS.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); - } - async close(): Promise { await this.pool.end(); } From ca2366cecfb8cb2d402f92858b0fe9fbe740442f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 21:28:50 +0200 Subject: [PATCH 40/85] Schema check: validity is the rule; pin the printed predicates The mutations dropping indisvalid or indisready each survived alone, because a failed CREATE INDEX CONCURRENTLY clears both. Postgres never marks an index valid before it is ready (DROP INDEX CONCURRENTLY clears validity first), so indisready adds nothing and is deleted. A fixture now holds an index that is ready but not valid, the state an interrupted DROP INDEX CONCURRENTLY leaves, set directly in pg_index by the admin role. Comparing the raw declared predicate instead of Postgres's printing of it survived too: the schema's one predicate splits into the same conditions either way. A live test now pins printedDeclaredPredicates to the printed predicate of the applied attempt index, so the printing itself is checked. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 10 +++++----- tests/store.test.ts | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index fc9e687..e589f2e 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -370,7 +370,7 @@ export function samePredicate(declared: string, actual: string, keyColumns: stri * it with the declared predicate, reads the printed form and rolls back. * The names and SQL are the schema's own constants, never input. */ -async function printedDeclaredPredicates(pool: Pick): Promise { +export async function printedDeclaredPredicates(pool: Pick): Promise { const partial = REQUIRED_KEYS.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); if (partial.length === 0) return REQUIRED_KEYS.map(() => ""); const statements = partial.flatMap(({ table, columns, predicate, index }) => { @@ -426,9 +426,10 @@ export class PostgresStore implements HeimdallStore { * columns. A missing or altered piece fails with code SCHEMA_MISSING and a * missing privilege with SCHEMA_PRIVILEGES; neither error names the piece. * - * A key is met by a unique index on its table that is valid and ready (a - * failed CREATE INDEX CONCURRENTLY leaves one that is neither), immediate - * (ON CONFLICT refuses a deferrable arbiter), has exactly the key's columns + * A key is met by a unique index on its table that is valid (a failed + * CREATE INDEX CONCURRENTLY or an interrupted DROP INDEX CONCURRENTLY leaves + * one that is not; Postgres never marks an index valid before it is ready), + * immediate (ON CONFLICT refuses a deferrable arbiter), has exactly the key's columns * as its key columns (INCLUDE columns aside), compares them under * deterministic collations (a case-insensitive one would merge two * providers' user ids into one row), and has the key's predicate. A @@ -473,7 +474,6 @@ export class PostgresStore implements HeimdallStore { JOIN pg_index i ON i.indrelid = to_regclass(required.table_name) WHERE i.indisunique AND i.indisvalid - AND i.indisready AND i.indimmediate AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) AND ( diff --git a/tests/store.test.ts b/tests/store.test.ts index 22a694b..a94f76f 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -8,6 +8,7 @@ import { createPostgresStore, PostgresStore, predicateConditions, + printedDeclaredPredicates, samePredicate, STORE_TABLE_PRIVILEGES, } from "../src/store/postgres.js"; @@ -516,6 +517,17 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD expect(await start(urlFor(name, role, rolePassword), true)).toBe(refusal("42501")); }); + it("prints each declared predicate exactly as Postgres prints the applied index's", async () => { + const { owner } = await appliedDatabase(); + const printed = await printedDeclaredPredicates(owner); + const applied = await owner.query<{ predicate: string }>( + "SELECT pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'auth_completions_attempt_unconsumed_unique_idx'::regclass" + ); + const attemptKey = REQUIRED_KEYS.findIndex((key) => key.predicate !== ""); + expect(printed[attemptKey]).toBe(applied.rows[0]!.predicate); + expect(printed.filter((predicate) => predicate !== "")).toHaveLength(1); + }); + it("refuses a table whose primary key was dropped, even after applying the schema", async () => { const { name, owner } = await appliedDatabase(); await owner.query("ALTER TABLE auth_completions DROP CONSTRAINT auth_completions_pkey"); @@ -580,6 +592,12 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD "DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "INSERT INTO auth_completions (code, attempt_id, app_slug, provider, mode, account_id, session_id, return_to, payload_json, created_at, expires_at) VALUES ('c1', 'att', 'ghostlight', 'discord', 'sign_in', 'acc', 's', 'https://x/', '{}', now(), now() + interval '1 hour'), ('c2', 'att', 'ghostlight', 'discord', 'sign_in', 'acc', 's', 'https://x/', '{}', now(), now() + interval '1 hour')", "!CREATE UNIQUE INDEX CONCURRENTLY auth_completions_attempt_unconsumed_unique_idx ON auth_completions(app_slug, attempt_id) WHERE attempt_id IS NOT NULL AND consumed_at IS NULL"]], + // An interrupted DROP INDEX CONCURRENTLY leaves an index ready but not + // valid; the fixture sets that state directly (the admin role may). + ["a key left only as an index that is ready but not valid", + ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "CREATE UNIQUE INDEX li_half_dropped ON linked_identities(provider, provider_user_id)", + "UPDATE pg_index SET indisvalid = false WHERE indexrelid = 'li_half_dropped'::regclass"]], ["a key compared under a case-insensitive collation (two users' ids become one row)", ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", From d029e08a2e55f43a45f1d85557f48b6ae2128d9a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 22:52:40 +0200 Subject: [PATCH 41/85] Recipe test: explore loadConfig as Idunn starts Heimdall Soul pass 7, S7-1. The exploration always called loadConfig with no arguments, and the candidate socket alone is refused (it needs the base URL), so neither the Idunn candidate path nor --state-root was ever run: a secret read only there (Y6, Y7) went unseen. The recipe's [service] arguments after the entry script, each binding given a probe path, are now the argv of an Idunn-shaped run, together with a loopback candidate socket and its base URL. That run is added on its own and to every combined run. A test pins the parsed argv, that both environment names are declared in the recipe, and that loadConfig accepts the shape. Co-Authored-By: Claude Opus 5.5 --- tests/recipe-secrets.test.ts | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/recipe-secrets.test.ts b/tests/recipe-secrets.test.ts index d7f69ec..ea144c5 100644 --- a/tests/recipe-secrets.test.ts +++ b/tests/recipe-secrets.test.ts @@ -42,7 +42,10 @@ const everyDeclaredName = [...recipe.replace(/\r\n/g, "\n").matchAll(/^\w+_envir // secret found so far set in plaintext; then each name read so far set to // each probe value (a few shapes, plus every string literal in config.ts); // then every name at once, each set to a value loadConfig accepted for it -// alone. A run that throws still counts the reads it made first. +// alone; and every combined run again as Idunn starts Heimdall, with the +// recipe's service arguments and a candidate socket with its base URL, a +// pair loadConfig refuses one name at a time. A run that throws still counts +// the reads it made first. // // Known limit: a read gated on a value that is none of the probe values and // not accepted alone in combination with the others stays unseen. @@ -83,17 +86,31 @@ async function withRecordingProcessEnv(run: () => T | Promise): Promise } } +/** + * How Idunn starts Heimdall: the recipe's [service] arguments after the + * entry script, each binding given a probe value, and the environment an + * Idunn candidate always has (a loopback socket, which requires the base URL). + */ +const recipeArguments = [...(/^\[service\][\s\S]*?^arguments = \[([\s\S]*?)^\]/m.exec(recipe.replace(/\r\n/g, "\n"))?.[1] ?? "").matchAll( + /\{ kind = "(literal|binding)", (?:value|name) = "([^"]+)" \}/g +)].map((match) => (match[1] === "literal" ? match[2]! : "/probe/idunn-binding")); +const idunnArgv = recipeArguments.slice(1); +const idunnEnv = { GAMECULT_IDUNN_CANDIDATE_BIND: "127.0.0.1:4100", GC_ACCESS_BASE_URL: "https://probe.test" }; + vi.resetModules(); const { loadConfig } = await withRecordingProcessEnv(() => import("../src/config.js")); /** The names one loadConfig run reads under `values`, and whether it accepted them. */ -async function readsOf(values: Readonly>): Promise<{ names: Set; accepted: boolean }> { +async function readsOf( + values: Readonly>, + argv: readonly string[] = [] +): Promise<{ names: Set; accepted: boolean }> { recorder.values = { ...values }; recorder.names = new Set(); let accepted = true; await withRecordingProcessEnv(() => { try { - loadConfig(recordingEnv, []); + loadConfig(recordingEnv, argv); } catch { accepted = false; // the reads before the refusal still count } @@ -113,6 +130,7 @@ async function exploreLoadConfig(): Promise<{ names: Set; enumerated: bo const keep = (run: { names: Set }) => run.names.forEach((name) => names.add(name)); keep(await readsOf({})); keep(await readsOf(secrets)); + keep(await readsOf({ ...secrets, ...idunnEnv }, idunnArgv)); for (const name of settable) { for (const value of probeValues) { const run = await readsOf({ ...secrets, [name]: value }); @@ -124,6 +142,7 @@ async function exploreLoadConfig(): Promise<{ names: Set; enumerated: bo for (let round = 0; round < widest; round += 1) { const together = Object.fromEntries([...accepted].map(([name, values]) => [name, values[round % values.length]!])); keep(await readsOf({ ...secrets, ...together })); + keep(await readsOf({ ...secrets, ...together, ...idunnEnv }, idunnArgv)); } grew = names.size > before; } @@ -161,6 +180,12 @@ describe("Idunn recipe secret declarations", () => { expect(plaintext).toEqual([]); }); + it("starts the exploration the way the recipe starts Heimdall", async () => { + expect(idunnArgv).toEqual(["--state-root", "/probe/idunn-binding"]); + expect(serviceNames).toEqual(expect.arrayContaining(Object.keys(idunnEnv))); + expect((await readsOf({ ...idunnEnv }, idunnArgv)).accepted).toBe(true); + }); + it("reads the environment only by name, never by copying or enumerating it", () => { expect(explored.enumerated).toBe(false); }); From 588739857a441731b5a7baa53d1bf2ef9c409445 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 22:52:40 +0200 Subject: [PATCH 42/85] Schema: name what the startup check does not prove Soul pass 7, S7-2. The out-of-scope list named CHECK constraints, triggers and row-level security. It now also names unique indexes beyond or stricter than the required keys (including leftovers of an interrupted concurrent build, which are enforced once ready), NOT NULL columns the schema does not declare and that have no default, and column-level SELECT under SELECT * or RETURNING *, which needs every column of the table. Documentation only. Co-Authored-By: Claude Opus 5.5 --- src/store/schema.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/store/schema.ts b/src/store/schema.ts index f3588c0..420f5e6 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -187,8 +187,15 @@ function keyColumns(line: string, keyword: "PRIMARY KEY" | "UNIQUE"): string | u * speed lookups are not required; the unique ones are, by shape, in * REQUIRED_KEYS. * - * Out of scope: CHECK constraints, triggers and row-level security. Each can - * make a write fail on a database that passes this check. + * Out of scope, each able to make a write fail on a database that passes the + * check: CHECK constraints; triggers; row-level security; unique indexes + * beyond the required keys, or stricter than them, including the leftovers of + * an interrupted concurrent index build (a valid extra one refuses rows the + * store expects to write, and even an invalid one that is ready is enforced + * on writes); NOT NULL columns the schema does not declare and that have no + * default, which the store's INSERTs leave out; and SELECT granted column by + * column when a statement reads `SELECT *` or `RETURNING *`, which needs + * SELECT on every column the table has, not only the declared ones. */ /** Every table the schema creates. */ From 996f21f58cd60fdcbad0f02323ca77cf6e476c03 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 22:52:41 +0200 Subject: [PATCH 43/85] Schema check: deterministic column collations, a named TEMP refusal, and kills for the printer Soul pass 7, S7-3 and S7-4, P7-3 and P7-5. Columns (S7-3): every declared column must be under a deterministic collation. A case-insensitive provider_user_id whose key index compares under "C" passed, and the store's own lookups then matched other users' ids. The check becomes checkSchemaRequirements(pool, requirements), with Heimdall's store calling it with HEIMDALL_SCHEMA_REQUIREMENTS (its tables, columns, keys and privileges). That seam lets a live test check a predicate Postgres prints differently from how it is written (a literal gains its cast): the check must still pass, so comparing the raw text instead of the printed form (P7-3's Y1) now fails. A role that may not create temporary tables was refused with 42501 and told to check GC_ACCESS_DATABASE_URL_FILE (P7-5). The predicate printer now refuses it as SCHEMA_TEMP_PRIVILEGE, and startup says to grant the role TEMPORARY on its database, in fixed text. New fixtures (S7-4): a case-insensitive collation on a key's first column (Y3), NOT NULL added to a nullable column of linked_identities (Y5), and every privilege on auth_completions except on attempt_id (Y8) are refused; a key index that INCLUDEs an undeclared case-insensitive column starts (Y4). Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 10 +- src/store/postgres.ts | 302 +++++++++++++++++++++++------------------- tests/store.test.ts | 70 +++++++++- 3 files changed, 242 insertions(+), 140 deletions(-) diff --git a/src/store/index.ts b/src/store/index.ts index 972bd8f..69abbe6 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -32,9 +32,13 @@ export async function createStore(config: HeimdallConfig): Promise undefined); - throw new Error( - `Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); check GC_ACCESS_DATABASE_URL_FILE.` - ); + // A role that may not create temporary tables is not a bad URL, so that + // refusal names the privilege instead. + const remedy = + code === "SCHEMA_TEMP_PRIVILEGE" + ? "grant the database role TEMPORARY on its database" + : "check GC_ACCESS_DATABASE_URL_FILE"; + throw new Error(`Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); ${remedy}.`); } return store; diff --git a/src/store/postgres.ts b/src/store/postgres.ts index e589f2e..64f013d 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -363,36 +363,6 @@ export function samePredicate(declared: string, actual: string, keyColumns: stri return left.length === right.length && left.every((condition, index) => condition === right[index]); } -/** - * Each REQUIRED_KEYS predicate as Postgres prints it back (empty for none). - * One simple-protocol round trip on one connection: a transaction that - * builds a temporary copy of each keyed table's declared columns, indexes - * it with the declared predicate, reads the printed form and rolls back. - * The names and SQL are the schema's own constants, never input. - */ -export async function printedDeclaredPredicates(pool: Pick): Promise { - const partial = REQUIRED_KEYS.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); - if (partial.length === 0) return REQUIRED_KEYS.map(() => ""); - const statements = partial.flatMap(({ table, columns, predicate, index }) => { - const probe = `heimdall_key_probe_${index}`; - const declared = REQUIRED_COLUMNS.filter((column) => column.table === table) - .map((column) => `${column.column} ${column.type}`) - .join(", "); - return [ - `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, - `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, - ]; - }); - const select = partial - .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) - .join(" UNION ALL "); - const results = (await pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as Array<{ - rows: Array<{ key: number; predicate: string }>; - }>; - const printed = results[results.length - 2]!.rows; - return REQUIRED_KEYS.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); -} - /** * What the store's statements do to each table, and so the privileges its * database role needs: it reads what it returns or looks up, inserts what it @@ -411,6 +381,170 @@ export const STORE_TABLE_PRIVILEGES: Readonly>; +} + +/** What Heimdall's store needs, read from its schema and its statements. */ +export const HEIMDALL_SCHEMA_REQUIREMENTS: SchemaRequirements = { + tables: REQUIRED_TABLES, + columns: REQUIRED_COLUMNS, + keys: REQUIRED_KEYS, + privileges: STORE_TABLE_PRIVILEGES, +}; + +/** + * Each required key's predicate as Postgres prints it back (empty for none). + * One simple-protocol round trip on one connection: a transaction that builds + * a temporary copy of each keyed table's declared columns, indexes it with the + * declared predicate, reads the printed form and rolls back. The names and SQL + * come from the requirements, which are constants, never input. A role that + * may not create temporary tables fails with code SCHEMA_TEMP_PRIVILEGE. + */ +export async function printedDeclaredPredicates(pool: Pick, requirements: SchemaRequirements): Promise { + const partial = requirements.keys.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); + if (partial.length === 0) return requirements.keys.map(() => ""); + const statements = partial.flatMap(({ table, columns, predicate, index }) => { + const probe = `heimdall_key_probe_${index}`; + const declared = requirements.columns + .filter((column) => column.table === table) + .map((column) => `${column.column} ${column.type}`) + .join(", "); + return [ + `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, + `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, + ]; + }); + const select = partial + .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) + .join(" UNION ALL "); + let results: Array<{ rows: Array<{ key: number; predicate: string }> }>; + try { + results = (await pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as typeof results; + } catch (error) { + if ((error as { code?: unknown }).code === "42501") { + throw Object.assign(new Error("The Heimdall database role may not create temporary tables."), { code: "SCHEMA_TEMP_PRIVILEGE" }); + } + throw error; + } + const printed = results[results.length - 2]!.rows; + return requirements.keys.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); +} + +/** + * Proves that the database can take a store's requests: every required + * table; every column with its declared type, under a deterministic + * collation, and nullable where declared nullable; every unique key; and + * every privilege, held on the table or on each of its columns. A missing or + * altered piece fails with code SCHEMA_MISSING and a missing privilege with + * SCHEMA_PRIVILEGES; neither error names the piece. + * + * A key is met by a unique index on its table that is valid (a failed + * CREATE INDEX CONCURRENTLY or an interrupted DROP INDEX CONCURRENTLY leaves + * one that is not; Postgres never marks an index valid before it is ready), + * immediate (ON CONFLICT refuses a deferrable arbiter), has exactly the key's + * columns as its key columns (INCLUDE columns aside), compares them under + * deterministic collations (a case-insensitive one would merge two + * providers' user ids into one row), and has the key's predicate. A + * predicate matches when both, as Postgres prints them, are the same set of + * AND-ed conditions; a `key_column IS NOT NULL` condition may be absent on + * either side, because a unique index never makes NULL keys collide unless + * it is NULLS NOT DISTINCT. Postgres prints the declared predicates itself + * (printedDeclaredPredicates). + * + * What this does not prove is listed where the schema is: see schema.ts. + */ +export async function checkSchemaRequirements(pool: Pick, requirements: SchemaRequirements): Promise { + const declaredPredicates = await printedDeclaredPredicates(pool, requirements); + const privileges = Object.entries(requirements.privileges).flatMap(([table, wanted]) => + wanted.map((privilege) => ({ table, privilege })) + ); + const result = await pool.query<{ + kind: "missing" | "privilege" | "candidate"; + key: number | null; + predicate: string | null; + nulls_not_distinct: boolean | null; + }>( + ` + SELECT 'missing' AS kind, NULL::int AS key, NULL::text AS predicate, NULL::boolean AS nulls_not_distinct + FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL + UNION ALL + SELECT 'missing', NULL, NULL, NULL + FROM unnest($2::text[], $3::text[], $4::text[], $5::boolean[]) AS required(table_name, column_name, type_name, nullable) + WHERE NOT EXISTS ( + SELECT 1 FROM pg_attribute a + WHERE a.attrelid = to_regclass(required.table_name) + AND a.attname = required.column_name + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.atttypid = to_regtype(required.type_name) + AND (NOT required.nullable OR NOT a.attnotnull) + AND NOT EXISTS (SELECT 1 FROM pg_collation c WHERE c.oid = a.attcollation AND NOT c.collisdeterministic) + ) + UNION ALL + SELECT 'candidate', required.key::int, pg_get_expr(i.indpred, i.indrelid), + coalesce((to_jsonb(i) ->> 'indnullsnotdistinct')::boolean, false) + FROM unnest($6::text[], $7::text[]) WITH ORDINALITY AS required(table_name, columns, key) + JOIN pg_index i ON i.indrelid = to_regclass(required.table_name) + WHERE i.indisunique + AND i.indisvalid + AND i.indimmediate + AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) + AND ( + SELECT string_agg(a.attname, ',' ORDER BY a.attname) + FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum + WHERE key.position <= i.indnkeyatts + ) = required.columns + AND NOT EXISTS ( + SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS key_collation(oid, position) + JOIN pg_collation c ON c.oid = key_collation.oid + WHERE key_collation.position <= i.indnkeyatts AND NOT c.collisdeterministic + ) + UNION ALL + SELECT 'privilege', NULL, NULL, NULL + FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) + WHERE EXISTS ( + SELECT 1 FROM unnest($2::text[], $3::text[]) AS used(table_name, column_name) + JOIN pg_attribute a ON a.attrelid = to_regclass(used.table_name) AND a.attname = used.column_name AND NOT a.attisdropped + WHERE used.table_name = required.table_name + AND NOT has_column_privilege(a.attrelid, a.attnum, required.privilege) + ) + `, + [ + requirements.tables, + requirements.columns.map((required) => required.table), + requirements.columns.map((required) => required.column), + requirements.columns.map((required) => required.type), + requirements.columns.map((required) => required.nullable), + requirements.keys.map((required) => required.table), + requirements.keys.map((required) => required.columns), + privileges.map((required) => required.table), + privileges.map((required) => required.privilege), + ] + ); + const keyMet = requirements.keys.map((required, index) => + result.rows.some( + (row) => + row.kind === "candidate" && + row.key === index + 1 && + samePredicate(declaredPredicates[index]!, row.predicate ?? "", required.columns.split(","), row.nulls_not_distinct === true) + ) + ); + if (result.rows.some((row) => row.kind === "missing") || keyMet.includes(false)) { + throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); + } + if (result.rows.some((row) => row.kind === "privilege")) { + throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { + code: "SCHEMA_PRIVILEGES", + }); + } +} + export class PostgresStore implements HeimdallStore { constructor(private readonly pool: Pick) {} @@ -418,113 +552,9 @@ export class PostgresStore implements HeimdallStore { await this.pool.query(CREATE_SCHEMA_SQL); } - /** - * Proves, before Heimdall serves, that the database can take its requests: - * every table the schema makes; every column with its declared type, and - * nullable where the schema leaves it nullable; every unique key; and every - * privilege in STORE_TABLE_PRIVILEGES, held on the table or on each of its - * columns. A missing or altered piece fails with code SCHEMA_MISSING and a - * missing privilege with SCHEMA_PRIVILEGES; neither error names the piece. - * - * A key is met by a unique index on its table that is valid (a failed - * CREATE INDEX CONCURRENTLY or an interrupted DROP INDEX CONCURRENTLY leaves - * one that is not; Postgres never marks an index valid before it is ready), - * immediate (ON CONFLICT refuses a deferrable arbiter), has exactly the key's columns - * as its key columns (INCLUDE columns aside), compares them under - * deterministic collations (a case-insensitive one would merge two - * providers' user ids into one row), and has the key's predicate. A - * predicate matches when both, as Postgres prints them, are the same set of - * AND-ed conditions; a `key_column IS NOT NULL` condition may be absent on - * either side, because a unique index never makes NULL keys collide unless - * it is NULLS NOT DISTINCT. - * - * Postgres prints the declared predicates itself: they are indexed on a - * temporary copy of the table in a transaction that is rolled back. - */ + /** Proves, before Heimdall serves, that the database can take its requests (checkSchemaRequirements). */ async checkSchema(): Promise { - const declaredPredicates = await printedDeclaredPredicates(this.pool); - const privileges = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => - wanted.map((privilege) => ({ table, privilege })) - ); - const result = await this.pool.query<{ - kind: "missing" | "privilege" | "candidate"; - key: number | null; - predicate: string | null; - nulls_not_distinct: boolean | null; - }>( - ` - SELECT 'missing' AS kind, NULL::int AS key, NULL::text AS predicate, NULL::boolean AS nulls_not_distinct - FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL - UNION ALL - SELECT 'missing', NULL, NULL, NULL - FROM unnest($2::text[], $3::text[], $4::text[], $5::boolean[]) AS required(table_name, column_name, type_name, nullable) - WHERE NOT EXISTS ( - SELECT 1 FROM pg_attribute a - WHERE a.attrelid = to_regclass(required.table_name) - AND a.attname = required.column_name - AND a.attnum > 0 - AND NOT a.attisdropped - AND a.atttypid = to_regtype(required.type_name) - AND (NOT required.nullable OR NOT a.attnotnull) - ) - UNION ALL - SELECT 'candidate', required.key::int, pg_get_expr(i.indpred, i.indrelid), - coalesce((to_jsonb(i) ->> 'indnullsnotdistinct')::boolean, false) - FROM unnest($6::text[], $7::text[]) WITH ORDINALITY AS required(table_name, columns, key) - JOIN pg_index i ON i.indrelid = to_regclass(required.table_name) - WHERE i.indisunique - AND i.indisvalid - AND i.indimmediate - AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) - AND ( - SELECT string_agg(a.attname, ',' ORDER BY a.attname) - FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) - JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum - WHERE key.position <= i.indnkeyatts - ) = required.columns - AND NOT EXISTS ( - SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS key_collation(oid, position) - JOIN pg_collation c ON c.oid = key_collation.oid - WHERE key_collation.position <= i.indnkeyatts AND NOT c.collisdeterministic - ) - UNION ALL - SELECT 'privilege', NULL, NULL, NULL - FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) - WHERE EXISTS ( - SELECT 1 FROM unnest($2::text[], $3::text[]) AS used(table_name, column_name) - JOIN pg_attribute a ON a.attrelid = to_regclass(used.table_name) AND a.attname = used.column_name AND NOT a.attisdropped - WHERE used.table_name = required.table_name - AND NOT has_column_privilege(a.attrelid, a.attnum, required.privilege) - ) - `, - [ - REQUIRED_TABLES, - REQUIRED_COLUMNS.map((required) => required.table), - REQUIRED_COLUMNS.map((required) => required.column), - REQUIRED_COLUMNS.map((required) => required.type), - REQUIRED_COLUMNS.map((required) => required.nullable), - REQUIRED_KEYS.map((required) => required.table), - REQUIRED_KEYS.map((required) => required.columns), - privileges.map((required) => required.table), - privileges.map((required) => required.privilege), - ] - ); - const keyMet = REQUIRED_KEYS.map((required, index) => - result.rows.some( - (row) => - row.kind === "candidate" && - row.key === index + 1 && - samePredicate(declaredPredicates[index]!, row.predicate ?? "", required.columns.split(","), row.nulls_not_distinct === true) - ) - ); - if (result.rows.some((row) => row.kind === "missing") || keyMet.includes(false)) { - throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); - } - if (result.rows.some((row) => row.kind === "privilege")) { - throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { - code: "SCHEMA_PRIVILEGES", - }); - } + await checkSchemaRequirements(this.pool, HEIMDALL_SCHEMA_REQUIREMENTS); } async close(): Promise { diff --git a/tests/store.test.ts b/tests/store.test.ts index a94f76f..a7a45f5 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -7,6 +7,8 @@ import { createStore } from "../src/store/index.js"; import { createPostgresStore, PostgresStore, + checkSchemaRequirements, + HEIMDALL_SCHEMA_REQUIREMENTS, predicateConditions, printedDeclaredPredicates, samePredicate, @@ -519,7 +521,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD it("prints each declared predicate exactly as Postgres prints the applied index's", async () => { const { owner } = await appliedDatabase(); - const printed = await printedDeclaredPredicates(owner); + const printed = await printedDeclaredPredicates(owner, HEIMDALL_SCHEMA_REQUIREMENTS); const applied = await owner.query<{ predicate: string }>( "SELECT pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'auth_completions_attempt_unconsumed_unique_idx'::regclass" ); @@ -552,6 +554,51 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD await exerciseStore(new PostgresStore(pool)); }); + it("refuses a privilege missing on one column of another table", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, { ...STORE_TABLE_PRIVILEGES, auth_completions: [] }); + const columns = REQUIRED_COLUMNS.filter((column) => column.table === "auth_completions" && column.column !== "attempt_id") + .map((column) => column.column) + .join(", "); + for (const privilege of STORE_TABLE_PRIVILEGES.auth_completions!) { + await owner.query(`GRANT ${privilege} (${columns}) ON auth_completions TO ${role}`); + } + + expect(await start(urlFor(name, role, rolePassword), false)).toBe(refusal("SCHEMA_PRIVILEGES")); + }); + + it("names the TEMPORARY privilege when the role may not create temporary tables", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, STORE_TABLE_PRIVILEGES); + await owner.query(`REVOKE TEMPORARY ON DATABASE ${name} FROM PUBLIC`); + + expect(await start(urlFor(name, role, rolePassword), false)).toBe( + "Postgres storage could not be prepared (SCHEMA_TEMP_PRIVILEGE); grant the database role TEMPORARY on its database." + ); + }); + + // The schema's own predicate splits into the same conditions raw or + // printed, so only a predicate whose printed form differs (a literal gains + // its cast) shows that the check compares Postgres's printing. + it("matches a declared predicate that Postgres prints differently from how it is written", async () => { + const { owner } = await appliedDatabase(); + await owner.query("CREATE TABLE predicate_probe (id INTEGER NOT NULL, state TEXT, attempts INTEGER)"); + await owner.query("CREATE UNIQUE INDEX predicate_probe_live ON predicate_probe (id) WHERE state = 'live' AND attempts > 0"); + const requirements = { + tables: ["predicate_probe"], + columns: [ + { table: "predicate_probe", column: "id", type: "INTEGER", nullable: false }, + { table: "predicate_probe", column: "state", type: "TEXT", nullable: true }, + { table: "predicate_probe", column: "attempts", type: "INTEGER", nullable: true }, + ], + keys: [{ table: "predicate_probe", columns: "id", predicate: "state = 'live' AND attempts > 0" }], + privileges: {}, + }; + + expect((await printedDeclaredPredicates(owner, requirements))[0]).toBe("((state = 'live'::text) AND (attempts > 0))"); + await expect(checkSchemaRequirements(owner, requirements)).resolves.toBeUndefined(); + }); + it("refuses a privilege held on only some of a table's columns", async () => { const { name, owner } = await appliedDatabase(); await grant(owner, { ...STORE_TABLE_PRIVILEGES, audit_events: [] }); @@ -598,6 +645,18 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE UNIQUE INDEX li_half_dropped ON linked_identities(provider, provider_user_id)", "UPDATE pg_index SET indisvalid = false WHERE indexrelid = 'li_half_dropped'::regclass"]], + ["a key's first column compared under a case-insensitive collation", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "CREATE UNIQUE INDEX li_ci_first ON linked_identities(provider COLLATE heimdall_ci, provider_user_id)"]], + // Soul pass 7: the key index compares under "C", but the column itself is + // case-insensitive, so the store's own lookups by provider_user_id match + // other users' ids. + ["a column declared under a case-insensitive collation, its key index under C", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "ALTER TABLE linked_identities ALTER COLUMN provider_user_id TYPE text COLLATE heimdall_ci", + 'CREATE UNIQUE INDEX li_c ON linked_identities(provider, provider_user_id COLLATE "C")']], ["a key compared under a case-insensitive collation (two users' ids become one row)", ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", @@ -617,6 +676,8 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD ["a column retyped", ["ALTER TABLE sessions ALTER COLUMN access_revision TYPE TEXT"]], ["a created table's column dropped", ["ALTER TABLE auth_completions DROP COLUMN mode"]], ["NOT NULL added to a column the store leaves null", ["ALTER TABLE auth_completions ALTER COLUMN attempt_id SET NOT NULL"]], + ["NOT NULL added to a nullable column of another table", + ["UPDATE linked_identities SET username = ''", "ALTER TABLE linked_identities ALTER COLUMN username SET NOT NULL"]], ])("refuses %s", async (_damage, steps) => { const { name, owner } = await appliedDatabase(); for (const step of steps) { @@ -639,6 +700,13 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD ["the attempt index without its redundant attempt_id IS NOT NULL", ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX attempt_unconsumed ON auth_completions(app_slug, attempt_id) WHERE consumed_at IS NULL"]], ["a lookup index dropped", ["DROP INDEX audit_events_lookup_idx", "DROP INDEX auth_completions_lookup_idx"]], + // Only a key's own columns decide which rows collide; an INCLUDEd column's + // collation does not, whatever it is. + ["a key index that INCLUDEs an undeclared case-insensitive column", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities ADD COLUMN operator_note text COLLATE heimdall_ci", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "CREATE UNIQUE INDEX li_note ON linked_identities(provider, provider_user_id) INCLUDE (operator_note)"]], ])("accepts %s", async (_change, steps) => { const { name, owner } = await appliedDatabase(); for (const step of steps) await owner.query(step); From 7904e17cd267c046e2fa08119f7ea310cbf68a83 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 23:18:53 +0200 Subject: [PATCH 44/85] Pin the predicate splitter on a quoted identifier Soul pass 7's Y2, treating only single quotes as quoted text, survived: no test gave the splitter a double-quoted identifier. One case now holds an identifier containing " AND (" and checks it stays whole. Co-Authored-By: Claude Opus 5.5 --- tests/store.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/store.test.ts b/tests/store.test.ts index a7a45f5..8eeaeb1 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -269,6 +269,7 @@ describe("PostgresStore.checkSchema", () => { "consumed_at IS NULL", ]); expect(predicateConditions("(s = 'x AND (y'::text)")).toEqual(["s = 'x AND (y'::text"]); + expect(predicateConditions('(("odd AND (name" IS NULL) AND b)')).toEqual(['"odd AND (name" IS NULL', "b"]); expect(predicateConditions("")).toEqual([]); }); From 8066bcbbee2ff49878de431502221356de5501aa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 10:44:24 +0200 Subject: [PATCH 45/85] State: Joystick H1 is the active cut; Ghostlight canaries demoted behind it Points at campaign joystick:campaign:self; flags the 1086aee deploy fact as last-recorded (2026-08-22, ~100 commits behind HEAD); drops the stale E: path and repixelizer-venv rehydrate command from the handoff. Co-Authored-By: Claude Opus 5.5 --- notes/fresh-workspace-handoff.md | 14 +++++++++++++- state/branches.json | 9 +++++++++ state/map.yaml | 9 +++++---- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index a8502a3..31358f9 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -1,6 +1,6 @@ # Fresh Workspace Handoff -This is the re-entry packet for `E:\Projects\Heimdall`. +This is the re-entry packet for this repo (path: whatever the checkout is; it has moved drives before). It is intentionally short. Historical proof belongs in git history and the distilled `state/evidence.jsonl` ledger; the shared architecture belongs in @@ -25,8 +25,18 @@ Get-Content '.\state\evidence.jsonl' -Tail 8 Do not trust this file for the exact live HEAD. Always check git. +## Active Work + +Joystick H1 is the active, M1-critical cut (operator ruling 2026-10-18): +campaign `joystick:campaign:self`, target `joystick:target:r1`, branch +`eureka/joystick-h1`. Read the campaign and target in the typed mind; do not +re-derive them here. The Ghostlight KLTST canaries below stay open, behind H1. + ## Current Orientation +Deploy facts below are last-recorded, from 2026-08-22 (`1086aee`); HEAD has +moved ~100 commits past it. Check Yggdrasil, not this file, for what runs. + - Yggdrasil runs Heimdall `1086aee01169bf60e8a492b2740db1c6f3e8cabf` with CultLib `5cefa0db0079a8e3ee22f29d7b9e6e5aa60912a9` @@ -171,6 +181,8 @@ Do not trust this file for the exact live HEAD. Always check git. ## Next Real Move +Joystick H1 first (see Active Work). Everything below is the standing queue. + Do not continue implementation automatically from a rehydrate-only request. The Idunn/swarm health transport cut is live. Do not regress Heimdall to diff --git a/state/branches.json b/state/branches.json index ea9d340..643b64f 100644 --- a/state/branches.json +++ b/state/branches.json @@ -12,6 +12,15 @@ "docs/implementation-plan.md" ], "notes": "Current mainline. Repixelizer is the first auth-blank binding target; StreamPixels is the migration and boundary stress test." + }, + { + "id": "eureka/joystick-h1", + "hypothesis": "One locked refresh primitive plus a Joystick OAuth provider in Heimdall lets StreamPixels ingest Joystick without ever presenting a provider refresh token.", + "status": "active", + "artifacts": [ + "state/map.yaml" + ], + "notes": "Campaign joystick:campaign:self, target joystick:target:r1. Hands owns code in a separate worktree." } ] } diff --git a/state/map.yaml b/state/map.yaml index 71e57dc..3f8772a 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -59,13 +59,14 @@ data_flow: - reject per-request auth round-trip chatter active_subgoals: - - Pressure the deployed Ghostlight gamecult.heimdall.access integration with two- and eight-account human canaries. + - ACTIVE, M1-critical (operator ruling 2026-10-18): Joystick H1, campaign joystick:campaign:self (branch eureka/joystick-h1). Joystick OAuth provider, joystick as a streampixels viewer provider, one locked refresh primitive for every provider replacing refreshLinkedIdentityCredentialIfNeeded. + - OPEN, behind H1: pressure the deployed Ghostlight gamecult.heimdall.access integration with two- and eight-account human canaries. - Keep the shared architecture explicit and separate from app profiles. - Preserve the default boundary that app-domain data remains app-owned. - Align the plan to Heimdall as a Yggdrasil-hosted authority service. - Publish Heimdall daemon health and boundary state for Idunn/Odin without moving auth truth out of Heimdall. - Verify the Discord and Patreon Repixelizer auth paths with real provider credentials, stable signing, persistent storage, and live browser callback verification. - - Wire StreamPixels through Heimdall for Twitch/YouTube viewer identity and managed creator-provider credentials without moving StreamPixels domain state into Heimdall. + - Wire StreamPixels through Heimdall for Twitch/YouTube (Joystick joins under H1) viewer identity and managed creator-provider credentials without moving StreamPixels domain state into Heimdall. accepted_design: - `gamecult.heimdall.access` owns access-gate representation and browser navigation semantics; it has no access-grant or provider-state authority. @@ -125,5 +126,5 @@ rejected_paths: reason: reusing auth mechanics does not justify centralizing app-domain audience or product data current_status: - summary: Yggdrasil runs the gamecult.heimdall.access plugin and encrypted loopback begin, complete, refresh, and logout plane at 1086aee01169bf60e8a492b2740db1c6f3e8cabf. Odin returns all four redacted discovery records, Ghostlight resolves the boundary through Odin, and live anonymous auth.begin is accepted without exposing claims or secrets. - next_action: Run separate-account KLTST completion, refresh, logout, privacy, and stale-attempt canaries through the deployed Ghostlight Eve host; keep provider credential rotation on the operator list because prior tool output exposed OAuth secrets. + summary: Active work is Joystick H1 (campaign joystick:campaign:self). Last recorded Yggdrasil deploy (not verifiable from this repo; HEAD is far past it, see git): Yggdrasil runs the gamecult.heimdall.access plugin and encrypted loopback begin, complete, refresh, and logout plane at 1086aee01169bf60e8a492b2740db1c6f3e8cabf. Odin returns all four redacted discovery records, Ghostlight resolves the boundary through Odin, and live anonymous auth.begin is accepted without exposing claims or secrets. + next_action: Land Joystick H1 per campaign joystick:campaign:self and target joystick:target:r1 (worktree and branch eureka/joystick-h1). Then, still open, run separate-account KLTST completion, refresh, logout, privacy, and stale-attempt canaries through the deployed Ghostlight Eve host; keep provider credential rotation on the operator list because prior tool output exposed OAuth secrets. From 2108d1088b117f9644d9ae98a92c6603d2e797d7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 10:44:29 +0200 Subject: [PATCH 46/85] Handoff: rehydrate with plain python, not the repixelizer venv on E: Co-Authored-By: Claude Opus 5.5 --- notes/fresh-workspace-handoff.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 31358f9..4727d7f 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -12,8 +12,7 @@ distilled `state/evidence.jsonl` ledger; the shared architecture belongs in From the repo root: ```powershell -# This repo has no dedicated venv yet. Use any working Python 3.11+. -E:\Projects\repixelizer\.venv\Scripts\python.exe .\tools\heimdall_state.py status +python .\tools\heimdall_state.py status Get-Content '.\state\map.yaml' Get-Content '.\notes\fresh-workspace-handoff.md' Get-Content '.\docs\architecture.md' From 0dbcd98019eba61b320938d06bb90e6b410cccf0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:06:32 +0200 Subject: [PATCH 47/85] Collapse ten per-provider token functions into one descriptor-driven request Endpoint, client-auth method (RFC 7591 names) and rotation now live on ProviderDescriptor; requestProviderTokens is the only code that talks to a token endpoint. Failure labels are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/oauth.ts | 550 ++++----------------------- src/providers.ts | 23 ++ tests/provider-token-request.test.ts | 116 ++++++ 3 files changed, 203 insertions(+), 486 deletions(-) create mode 100644 tests/provider-token-request.test.ts diff --git a/src/oauth.ts b/src/oauth.ts index 33ae495..2d0020c 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -7,6 +7,7 @@ import { } from "./contracts.js"; import { type HeimdallConfig } from "./config.js"; import { entitlementFacts } from "./facts.js"; +import { providerCatalog } from "./providers.js"; import { type CreateEntitlementSnapshotInput } from "./store/types.js"; export interface OAuthTokenSet { @@ -49,6 +50,7 @@ export interface OAuthProviderRuntime { refreshAccessToken?(options: { config: HeimdallConfig; refreshToken: string; + signal?: AbortSignal; }): Promise; resolveIdentity(options: { config: HeimdallConfig; accessToken: string }): Promise; evaluateEntitlements(options: { @@ -106,512 +108,88 @@ function normalizeScopes(scope: string | string[] | undefined): string[] { return scope?.split(/\s+/).filter(Boolean) ?? []; } -async function exchangeDiscordAuthorizationCode(options: { - config: HeimdallConfig; - code: string; - redirectUri: string; -}): Promise { - const providerConfig = options.config.providers.discord; +export type TokenGrant = + | { grant_type: "authorization_code"; code: string; redirect_uri: string } + | { grant_type: "refresh_token"; refresh_token: string }; + +/** + * The one token request: endpoint, client authentication and rotation all come + * from the provider's descriptor. `basic` sends the client as an Authorization + * header and nothing in the body; `post` sends it in the body. A refresh token + * is returned only when the provider sent one; keeping or discarding the old + * one is the caller's rule (ProviderDescriptor.rotatesRefreshTokens). + */ +export async function requestProviderTokens( + provider: Provider, + config: HeimdallConfig, + grant: TokenGrant, + signal?: AbortSignal +): Promise { + const descriptor = providerCatalog[provider]; + const providerConfig = config.providers[provider]; if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Discord OAuth is not fully configured."); + throw new Error(`${descriptor.displayName} OAuth is not fully configured.`); } - const body = new URLSearchParams({ - grant_type: "authorization_code", - code: options.code, - redirect_uri: options.redirectUri, - }); - const basicAuth = Buffer.from(`${providerConfig.clientId}:${providerConfig.clientSecret}`).toString("base64"); - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in: number; - scope?: string | string[]; - }>( - "https://discord.com/api/v10/oauth2/token", - { - method: "POST", - headers: { - Authorization: `Basic ${basicAuth}`, - "Content-Type": "application/x-www-form-urlencoded", - }, - body, - }, - "Discord token exchange failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - expiresAt: new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(), - raw: tokenResponse as unknown as Record, + const body = new URLSearchParams(grant); + const headers: Record = { + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; + if (descriptor.tokenEndpointAuthMethod === "client_secret_basic") { + headers.Authorization = `Basic ${Buffer.from(`${providerConfig.clientId}:${providerConfig.clientSecret}`).toString("base64")}`; + } else { + body.set("client_id", providerConfig.clientId); + body.set("client_secret", providerConfig.clientSecret); } - return tokenSet; -} - -async function refreshDiscordAccessToken(options: { - config: HeimdallConfig; - refreshToken: string; -}): Promise { - const providerConfig = options.config.providers.discord; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Discord OAuth is not fully configured."); + const init: RequestInit = { method: "POST", headers, body }; + if (signal) { + init.signal = signal; } - - const basicAuth = Buffer.from(`${providerConfig.clientId}:${providerConfig.clientSecret}`).toString("base64"); + const label = grant.grant_type === "refresh_token" ? "refresh" : "exchange"; const tokenResponse = await fetchJson<{ access_token: string; refresh_token?: string; - token_type: string; - expires_in: number; - scope?: string | string[]; - }>( - "https://discord.com/api/v10/oauth2/token", - { - method: "POST", - headers: { - Authorization: `Basic ${basicAuth}`, - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: options.refreshToken, - }), - }, - "Discord token refresh failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - expiresAt: new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - return tokenSet; -} - -async function exchangePatreonAuthorizationCode(options: { - config: HeimdallConfig; - code: string; - redirectUri: string; -}): Promise { - const providerConfig = options.config.providers.patreon; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Patreon OAuth is not fully configured."); - } - - const body = new URLSearchParams({ - grant_type: "authorization_code", - code: options.code, - client_id: providerConfig.clientId, - client_secret: providerConfig.clientSecret, - redirect_uri: options.redirectUri, - }); - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; + token_type?: string; expires_in?: number; scope?: string | string[]; - }>( - "https://www.patreon.com/api/oauth2/token", - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body, - }, - "Patreon token exchange failed" - ); + }>(descriptor.tokenEndpoint, init, `${descriptor.displayName} token ${label} failed`); const tokenSet: OAuthTokenSet = { accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, + tokenType: tokenResponse.token_type ?? "Bearer", scope: normalizeScopes(tokenResponse.scope), raw: tokenResponse as unknown as Record, }; - if (tokenResponse.refresh_token) { tokenSet.refreshToken = tokenResponse.refresh_token; } - - if (tokenResponse.expires_in) { + if (typeof tokenResponse.expires_in === "number" && tokenResponse.expires_in > 0) { tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); } - return tokenSet; } -async function refreshPatreonAccessToken(options: { - config: HeimdallConfig; - refreshToken: string; -}): Promise { - const providerConfig = options.config.providers.patreon; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Patreon OAuth is not fully configured."); - } - - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - }>( - "https://www.patreon.com/api/oauth2/token", - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: options.refreshToken, - client_id: providerConfig.clientId, - client_secret: providerConfig.clientSecret, - }), - }, - "Patreon token refresh failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; +function sharedRuntimeExchange(provider: Provider): OAuthProviderRuntime["exchangeAuthorizationCode"] { + return (options) => + requestProviderTokens(provider, options.config, { + grant_type: "authorization_code", + code: options.code, + redirect_uri: options.redirectUri, + }); } -async function exchangeTwitchAuthorizationCode(options: { - config: HeimdallConfig; - code: string; - redirectUri: string; -}): Promise { - const providerConfig = options.config.providers.twitch; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Twitch OAuth is not fully configured."); - } - - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - }>( - "https://id.twitch.tv/oauth2/token", - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - client_id: providerConfig.clientId, - client_secret: providerConfig.clientSecret, - code: options.code, - grant_type: "authorization_code", - redirect_uri: options.redirectUri, - }), - }, - "Twitch token exchange failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; +function sharedRuntimeRefresh(provider: Provider): NonNullable { + return (options) => + requestProviderTokens( + provider, + options.config, + { grant_type: "refresh_token", refresh_token: options.refreshToken }, + options.signal + ); } -async function refreshTwitchAccessToken(options: { - config: HeimdallConfig; - refreshToken: string; -}): Promise { - const providerConfig = options.config.providers.twitch; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Twitch OAuth is not fully configured."); - } - - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - }>( - "https://id.twitch.tv/oauth2/token", - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - client_id: providerConfig.clientId, - client_secret: providerConfig.clientSecret, - grant_type: "refresh_token", - refresh_token: options.refreshToken, - }), - }, - "Twitch token refresh failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; -} - -async function exchangeYouTubeAuthorizationCode(options: { - config: HeimdallConfig; - code: string; - redirectUri: string; -}): Promise { - const providerConfig = options.config.providers.youtube; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("YouTube OAuth is not fully configured."); - } - - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - id_token?: string; - }>( - "https://oauth2.googleapis.com/token", - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - client_id: providerConfig.clientId, - client_secret: providerConfig.clientSecret, - code: options.code, - grant_type: "authorization_code", - redirect_uri: options.redirectUri, - }), - }, - "YouTube token exchange failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; -} - -async function refreshYouTubeAccessToken(options: { - config: HeimdallConfig; - refreshToken: string; -}): Promise { - const providerConfig = options.config.providers.youtube; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("YouTube OAuth is not fully configured."); - } - - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - }>( - "https://oauth2.googleapis.com/token", - { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - client_id: providerConfig.clientId, - client_secret: providerConfig.clientSecret, - grant_type: "refresh_token", - refresh_token: options.refreshToken, - }), - }, - "YouTube token refresh failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; -} - -async function exchangeSpotifyAuthorizationCode(options: { - config: HeimdallConfig; - code: string; - redirectUri: string; -}): Promise { - const providerConfig = options.config.providers.spotify; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Spotify OAuth is not fully configured."); - } - - const basicAuth = Buffer.from(`${providerConfig.clientId}:${providerConfig.clientSecret}`).toString("base64"); - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - }>( - "https://accounts.spotify.com/api/token", - { - method: "POST", - headers: { - Authorization: `Basic ${basicAuth}`, - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "authorization_code", - code: options.code, - redirect_uri: options.redirectUri, - }), - }, - "Spotify token exchange failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; -} - -async function refreshSpotifyAccessToken(options: { - config: HeimdallConfig; - refreshToken: string; -}): Promise { - const providerConfig = options.config.providers.spotify; - if (!providerConfig.clientId || !providerConfig.clientSecret) { - throw new Error("Spotify OAuth is not fully configured."); - } - - const basicAuth = Buffer.from(`${providerConfig.clientId}:${providerConfig.clientSecret}`).toString("base64"); - const tokenResponse = await fetchJson<{ - access_token: string; - refresh_token?: string; - token_type: string; - expires_in?: number; - scope?: string | string[]; - }>( - "https://accounts.spotify.com/api/token", - { - method: "POST", - headers: { - Authorization: `Basic ${basicAuth}`, - "Content-Type": "application/x-www-form-urlencoded", - }, - body: new URLSearchParams({ - grant_type: "refresh_token", - refresh_token: options.refreshToken, - }), - }, - "Spotify token refresh failed" - ); - - const tokenSet: OAuthTokenSet = { - accessToken: tokenResponse.access_token, - tokenType: tokenResponse.token_type, - scope: normalizeScopes(tokenResponse.scope), - raw: tokenResponse as unknown as Record, - }; - - if (tokenResponse.refresh_token) { - tokenSet.refreshToken = tokenResponse.refresh_token; - } - - if (tokenResponse.expires_in) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } - - return tokenSet; -} async function resolveDiscordIdentity(options: { accessToken: string }): Promise { const user = await fetchJson<{ @@ -1217,22 +795,22 @@ async function evaluatePatreonEntitlements(options: { } const discordRuntime: OAuthProviderRuntime = { - exchangeAuthorizationCode: exchangeDiscordAuthorizationCode, - refreshAccessToken: refreshDiscordAccessToken, + exchangeAuthorizationCode: sharedRuntimeExchange("discord"), + refreshAccessToken: sharedRuntimeRefresh("discord"), resolveIdentity: resolveDiscordIdentity, evaluateEntitlements: evaluateDiscordEntitlements, }; const patreonRuntime: OAuthProviderRuntime = { - exchangeAuthorizationCode: exchangePatreonAuthorizationCode, - refreshAccessToken: refreshPatreonAccessToken, + exchangeAuthorizationCode: sharedRuntimeExchange("patreon"), + refreshAccessToken: sharedRuntimeRefresh("patreon"), resolveIdentity: resolvePatreonIdentity, evaluateEntitlements: evaluatePatreonEntitlements, }; const twitchRuntime: OAuthProviderRuntime = { - exchangeAuthorizationCode: exchangeTwitchAuthorizationCode, - refreshAccessToken: refreshTwitchAccessToken, + exchangeAuthorizationCode: sharedRuntimeExchange("twitch"), + refreshAccessToken: sharedRuntimeRefresh("twitch"), resolveIdentity: resolveTwitchIdentity, async evaluateEntitlements() { return { facts: [], snapshots: [] }; @@ -1240,8 +818,8 @@ const twitchRuntime: OAuthProviderRuntime = { }; const youtubeRuntime: OAuthProviderRuntime = { - exchangeAuthorizationCode: exchangeYouTubeAuthorizationCode, - refreshAccessToken: refreshYouTubeAccessToken, + exchangeAuthorizationCode: sharedRuntimeExchange("youtube"), + refreshAccessToken: sharedRuntimeRefresh("youtube"), resolveIdentity: resolveYouTubeIdentity, async evaluateEntitlements() { return { facts: [], snapshots: [] }; @@ -1249,8 +827,8 @@ const youtubeRuntime: OAuthProviderRuntime = { }; const spotifyRuntime: OAuthProviderRuntime = { - exchangeAuthorizationCode: exchangeSpotifyAuthorizationCode, - refreshAccessToken: refreshSpotifyAccessToken, + exchangeAuthorizationCode: sharedRuntimeExchange("spotify"), + refreshAccessToken: sharedRuntimeRefresh("spotify"), resolveIdentity: resolveSpotifyIdentity, async evaluateEntitlements() { return { facts: [], snapshots: [] }; diff --git a/src/providers.ts b/src/providers.ts index 506e084..2998e4d 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -6,6 +6,11 @@ export interface ProviderDescriptor { key: Provider; displayName: string; authorizationEndpoint: string; + tokenEndpoint: string; + /** RFC 7591 names: how the client authenticates at the token endpoint. */ + tokenEndpointAuthMethod: "client_secret_basic" | "client_secret_post"; + /** True when a refresh response replaces the refresh token and the old one dies. */ + rotatesRefreshTokens: boolean; defaultScopes: string[]; roles: ProviderRole[]; clientIdEnv: string; @@ -18,6 +23,9 @@ export const providerCatalog: Record = { key: "discord", displayName: "Discord", authorizationEndpoint: "https://discord.com/oauth2/authorize", + tokenEndpoint: "https://discord.com/api/v10/oauth2/token", + tokenEndpointAuthMethod: "client_secret_basic", + rotatesRefreshTokens: false, defaultScopes: ["identify", "guilds.members.read"], roles: ["identity", "entitlement"], clientIdEnv: "GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID", @@ -27,6 +35,9 @@ export const providerCatalog: Record = { key: "patreon", displayName: "Patreon", authorizationEndpoint: "https://www.patreon.com/oauth2/authorize", + tokenEndpoint: "https://www.patreon.com/api/oauth2/token", + tokenEndpointAuthMethod: "client_secret_post", + rotatesRefreshTokens: false, defaultScopes: ["identity", "identity[email]"], roles: ["identity", "entitlement"], clientIdEnv: "GC_ACCESS_PROVIDER_PATREON_CLIENT_ID", @@ -36,6 +47,9 @@ export const providerCatalog: Record = { key: "github", displayName: "GitHub", authorizationEndpoint: "https://github.com/login/oauth/authorize", + tokenEndpoint: "https://github.com/login/oauth/access_token", + tokenEndpointAuthMethod: "client_secret_post", + rotatesRefreshTokens: false, defaultScopes: ["read:user", "user:email"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_GITHUB_CLIENT_ID", @@ -45,6 +59,9 @@ export const providerCatalog: Record = { key: "twitch", displayName: "Twitch", authorizationEndpoint: "https://id.twitch.tv/oauth2/authorize", + tokenEndpoint: "https://id.twitch.tv/oauth2/token", + tokenEndpointAuthMethod: "client_secret_post", + rotatesRefreshTokens: false, defaultScopes: ["user:read:email"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID", @@ -54,6 +71,9 @@ export const providerCatalog: Record = { key: "youtube", displayName: "YouTube", authorizationEndpoint: "https://accounts.google.com/o/oauth2/v2/auth", + tokenEndpoint: "https://oauth2.googleapis.com/token", + tokenEndpointAuthMethod: "client_secret_post", + rotatesRefreshTokens: false, defaultScopes: [ "openid", "email", @@ -73,6 +93,9 @@ export const providerCatalog: Record = { key: "spotify", displayName: "Spotify", authorizationEndpoint: "https://accounts.spotify.com/authorize", + tokenEndpoint: "https://accounts.spotify.com/api/token", + tokenEndpointAuthMethod: "client_secret_basic", + rotatesRefreshTokens: false, defaultScopes: [ "user-read-currently-playing", "user-read-playback-state", diff --git a/tests/provider-token-request.test.ts b/tests/provider-token-request.test.ts new file mode 100644 index 0000000..2d13a30 --- /dev/null +++ b/tests/provider-token-request.test.ts @@ -0,0 +1,116 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { type HeimdallConfig } from "../src/config.js"; +import { providers, type Provider } from "../src/contracts.js"; +import { requestProviderTokens } from "../src/oauth.js"; +import { providerCatalog } from "../src/providers.js"; + +const originalFetch = globalThis.fetch; +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function configFor(): HeimdallConfig { + return { + providers: Object.fromEntries(providers.map((p) => [p, { clientId: `${p}-client`, clientSecret: `${p}-secret` }])), + } as unknown as HeimdallConfig; +} + +interface Captured { + url: string; + headers: Record; + body: URLSearchParams; +} + +function capture(answer: () => Response): { calls: Captured[] } { + const calls: Captured[] = []; + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + calls.push({ + url: String(input), + headers: init?.headers as Record, + body: init?.body as URLSearchParams, + }); + return answer(); + }) as typeof fetch; + return { calls }; +} + +const ok = () => + new Response(JSON.stringify({ access_token: "a", refresh_token: "r", token_type: "Bearer", expires_in: 3600, scope: "x y" }), { + status: 200, + }); + +const requestable: Provider[] = [...providers]; + +describe("requestProviderTokens follows the provider descriptor", () => { + it.each(requestable.flatMap((p) => [[p, "authorization_code"] as const, [p, "refresh_token"] as const]))( + "%s %s hits its token endpoint with its client-auth method only", + async (provider, grantType) => { + const { calls } = capture(ok); + const descriptor = providerCatalog[provider]; + const grant = + grantType === "authorization_code" + ? ({ grant_type: "authorization_code", code: "the-code", redirect_uri: "https://h/cb" } as const) + : ({ grant_type: "refresh_token", refresh_token: "the-refresh" } as const); + + const tokens = await requestProviderTokens(provider, configFor(), grant); + + expect(calls).toHaveLength(1); + const call = calls[0]!; + expect(call.url).toBe(descriptor.tokenEndpoint); + expect(call.headers["Content-Type"]).toBe("application/x-www-form-urlencoded"); + expect(call.headers.Accept).toBe("application/json"); + expect(call.body.get("grant_type")).toBe(grantType); + if (descriptor.tokenEndpointAuthMethod === "client_secret_basic") { + expect(call.headers.Authorization).toBe(`Basic ${Buffer.from(`${provider}-client:${provider}-secret`).toString("base64")}`); + expect(call.body.has("client_id")).toBe(false); + expect(call.body.has("client_secret")).toBe(false); + } else { + expect(call.headers.Authorization).toBeUndefined(); + expect(call.body.get("client_id")).toBe(`${provider}-client`); + expect(call.body.get("client_secret")).toBe(`${provider}-secret`); + } + expect(tokens.accessToken).toBe("a"); + expect(tokens.refreshToken).toBe("r"); + expect(tokens.scope).toEqual(["x", "y"]); + expect(Date.parse(tokens.expiresAt ?? "")).toBeGreaterThan(Date.now()); + } + ); + + it("sends the code and redirect uri on exchange and the refresh token on refresh", async () => { + const { calls } = capture(ok); + await requestProviderTokens("spotify", configFor(), { grant_type: "authorization_code", code: "C", redirect_uri: "https://h/cb" }); + await requestProviderTokens("spotify", configFor(), { grant_type: "refresh_token", refresh_token: "R" }); + expect(calls[0]!.body.get("code")).toBe("C"); + expect(calls[0]!.body.get("redirect_uri")).toBe("https://h/cb"); + expect(calls[1]!.body.get("refresh_token")).toBe("R"); + expect(calls[1]!.body.has("code")).toBe(false); + }); + + it.each([ + ["no expires_in", { access_token: "a" }], + ["a zero expires_in", { access_token: "a", expires_in: 0 }], + ["a string expires_in", { access_token: "a", expires_in: "3600" }], + ])("sets no expiry and no refresh token for %s", async (_n, body) => { + capture(() => new Response(JSON.stringify(body), { status: 200 })); + const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "R" }); + expect(tokens.expiresAt).toBeUndefined(); + expect(tokens.refreshToken).toBeUndefined(); + }); + + it.each([ + ["authorization_code", { grant_type: "authorization_code", code: "c", redirect_uri: "u" } as const, "exchange"], + ["refresh_token", { grant_type: "refresh_token", refresh_token: "r" } as const, "refresh"], + ])("keeps the failure label for %s", async (_g, grant, word) => { + capture(() => new Response("nope", { status: 500 })); + await expect(requestProviderTokens("youtube", configFor(), grant)).rejects.toThrow( + `YouTube token ${word} failed (status 500, provider_status).` + ); + }); + + it("refuses an unconfigured provider by display name", async () => { + const config = { providers: { patreon: {} } } as unknown as HeimdallConfig; + await expect( + requestProviderTokens("patreon", config, { grant_type: "refresh_token", refresh_token: "r" }) + ).rejects.toThrow("Patreon OAuth is not fully configured."); + }); +}); From c81f9238907fa34f7f53d83c93c9facdf89dd82f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:11:34 +0200 Subject: [PATCH 48/85] Refresh managed credentials under a row lock, with typed revocation refreshLinkedIdentityCredentialIfNeeded is deleted. refreshManagedCredential owns the policy; HeimdallStore.updateLinkedIdentityCredential owns the row (SELECT ... FOR UPDATE in Postgres, a per-id chain in memory), so a refresh token is presented once across concurrent requests and processes. invalid_grant clears the tokens (410 managed_credential_revoked); any other failure keeps them (502 managed_credential_refresh_failed). The "?? refreshToken" keep-old fallback is gone; keeping it is an explicit branch for non-rotating providers. Co-Authored-By: Claude Opus 5.5 --- src/app.ts | 118 ++---- src/managed-credential.ts | 140 ++++++++ src/oauth.ts | 29 +- src/store/in-memory.ts | 42 +++ src/store/postgres.ts | 57 ++- src/store/types.ts | 28 ++ tests/app.test.ts | 113 +++++- tests/managed-credential.test.ts | 517 +++++++++++++++++++++++++++ tests/provider-token-request.test.ts | 31 ++ tests/store.test.ts | 3 +- 10 files changed, 979 insertions(+), 99 deletions(-) create mode 100644 src/managed-credential.ts create mode 100644 tests/managed-credential.test.ts diff --git a/src/app.ts b/src/app.ts index 9621856..5345c40 100644 --- a/src/app.ts +++ b/src/app.ts @@ -34,6 +34,7 @@ import { summarizePatreonMemberships, type OAuthRuntimeRegistry, } from "./oauth.js"; +import { ManagedCredentialRefreshFailed, refreshManagedCredential } from "./managed-credential.js"; import { buildAuthorizationUrl, providerCatalog, providerExpectedEnv } from "./providers.js"; import { createRuntimeKeyMaterial, signJwt, verifyJwt } from "./signing.js"; import { createStore, type HeimdallStore } from "./store/index.js"; @@ -44,8 +45,6 @@ import { type UpsertLinkedIdentityInput, } from "./store/types.js"; -const MANAGED_CREDENTIAL_REFRESH_SKEW_MS = 5 * 60 * 1000; - export interface BuildAppOptions { config?: HeimdallConfig; store?: HeimdallStore; @@ -367,63 +366,6 @@ function centsToDecimalString(cents: number): string { return (cents / 100).toFixed(2); } -async function refreshLinkedIdentityCredentialIfNeeded(options: { - config: HeimdallConfig; - store: HeimdallStore; - tokenCustody: TokenCustody; - oauthRuntimes: OAuthRuntimeRegistry; - linkedIdentity: StoredLinkedIdentity; - now: string; -}): Promise { - const expiresAt = options.linkedIdentity.tokenExpiresAt ? Date.parse(options.linkedIdentity.tokenExpiresAt) : undefined; - if ( - !expiresAt || - expiresAt - Date.now() > MANAGED_CREDENTIAL_REFRESH_SKEW_MS || - !options.linkedIdentity.refreshTokenEncrypted - ) { - return options.linkedIdentity; - } - - const runtime = options.oauthRuntimes[options.linkedIdentity.provider]; - if (!runtime.refreshAccessToken) { - return options.linkedIdentity; - } - - const refreshToken = options.tokenCustody.decrypt(options.linkedIdentity.refreshTokenEncrypted); - const refreshed = await runtime.refreshAccessToken({ - config: options.config, - refreshToken, - }); - - const upsertInput: UpsertLinkedIdentityInput = { - id: options.linkedIdentity.id, - accountId: options.linkedIdentity.accountId, - provider: options.linkedIdentity.provider, - providerUserId: options.linkedIdentity.providerUserId, - accessTokenEncrypted: options.tokenCustody.encrypt(refreshed.accessToken), - refreshTokenEncrypted: options.tokenCustody.encrypt(refreshed.refreshToken ?? refreshToken), - scopes: refreshed.scope.length ? refreshed.scope : options.linkedIdentity.scopes, - profileJson: options.linkedIdentity.profileJson, - createdAt: options.linkedIdentity.createdAt, - updatedAt: options.now, - }; - const tokenExpiresAt = refreshed.expiresAt ?? options.linkedIdentity.tokenExpiresAt; - if (tokenExpiresAt !== undefined) { - upsertInput.tokenExpiresAt = tokenExpiresAt; - } - if (options.linkedIdentity.username !== undefined) { - upsertInput.username = options.linkedIdentity.username; - } - if (options.linkedIdentity.displayName !== undefined) { - upsertInput.displayName = options.linkedIdentity.displayName; - } - if (options.linkedIdentity.primaryEmail !== undefined) { - upsertInput.primaryEmail = options.linkedIdentity.primaryEmail; - } - - return options.store.upsertLinkedIdentity(upsertInput); -} - function storedIdentityToResolvedIdentity(identity: StoredLinkedIdentity) { const resolved = { provider: identity.provider, @@ -450,7 +392,6 @@ async function evaluateStoredEntitlements(options: { appSlug: AppSlug; accountId: string; entitlementPolicies: OAuthEntitlementPolicy[]; - now: string; }): Promise { const profile = getAppProfile(options.appSlug); if (!profile) { @@ -460,19 +401,13 @@ async function evaluateStoredEntitlements(options: { const facts: string[] = []; for (const storedIdentity of linkedIdentities) { - if (!profile.entitlementSources.includes(storedIdentity.provider) || !storedIdentity.accessTokenEncrypted) { + if (!profile.entitlementSources.includes(storedIdentity.provider)) { continue; } - const linkedIdentity = await refreshLinkedIdentityCredentialIfNeeded({ - config: options.config, - store: options.store, - tokenCustody: options.tokenCustody, - oauthRuntimes: options.oauthRuntimes, - linkedIdentity: storedIdentity, - now: options.now, - }); - if (!linkedIdentity.accessTokenEncrypted) { + const refreshed = await refreshManagedCredential(options, storedIdentity); + const linkedIdentity = refreshed.identity; + if (refreshed.kind === "revoked" || !linkedIdentity.accessTokenEncrypted) { continue; } @@ -714,7 +649,6 @@ export async function refreshAppSession( appSlug, accountId: refreshClaim.account_id, entitlementPolicies, - now: nowIso, }); const issueInput: IssueAccessClaimInput = { appSlug, @@ -1620,21 +1554,15 @@ export async function buildApp(options: BuildAppOptions = {}): Promise number; + timeoutMs?: number; +} + +export type ManagedCredentialRevocationReason = "provider_revoked" | "rotation_missing"; + +/** `revoked` means the identity holds no access token: the provider took the grant back, or it was never held. */ +export type ManagedCredentialOutcome = + | { kind: "current"; identity: StoredLinkedIdentity } + | { kind: "revoked"; identity: StoredLinkedIdentity }; + +/** + * The provider could not be asked, or answered with something that is not a + * grant verdict (invalid_client, 5xx, timeout, malformed body). The stored + * tokens are untouched. Carries the provider and nothing the provider said. + */ +export class ManagedCredentialRefreshFailed extends Error { + readonly code = "managed_credential_refresh_failed"; + readonly provider: Provider; + + constructor(provider: Provider) { + super(`${providerCatalog[provider].displayName} managed credential refresh failed.`); + this.name = "ManagedCredentialRefreshFailed"; + this.provider = provider; + } +} + +function dueForRefresh(identity: StoredLinkedIdentity, runtime: OAuthProviderRuntime, now: number): boolean { + if (!identity.tokenExpiresAt || !identity.refreshTokenEncrypted || !runtime.refreshAccessToken) { + return false; + } + return Date.parse(identity.tokenExpiresAt) - now <= MANAGED_CREDENTIAL_REFRESH_SKEW_MS; +} + +function withoutTokens(identity: StoredLinkedIdentity): StoredLinkedIdentity { + const { accessTokenEncrypted: _a, refreshTokenEncrypted: _r, tokenExpiresAt: _e, ...rest } = identity; + return rest; +} + +/** + * The one refresh policy. `snapshot` is a hint: it may skip the lock when no + * refresh is due and never decides one. The row's lock owns when a refresh + * token is presented; the decision is made on the row as read under it. + */ +export async function refreshManagedCredential( + ctx: ManagedCredentialContext, + snapshot: StoredLinkedIdentity +): Promise { + const clock = ctx.clock ?? Date.now; + const provider = snapshot.provider; + const runtime = ctx.oauthRuntimes[provider]; + + if (!snapshot.accessTokenEncrypted) { + return { kind: "revoked", identity: snapshot }; + } + if (!dueForRefresh(snapshot, runtime, clock())) { + return { kind: "current", identity: snapshot }; + } + + let revocation: ManagedCredentialRevocationReason | undefined; + const decide = async (locked: StoredLinkedIdentity): Promise => { + if (!locked.accessTokenEncrypted || !dueForRefresh(locked, runtime, clock())) { + return { kind: "keep" }; + } + const updatedAt = new Date(clock()).toISOString(); + const refreshToken = ctx.tokenCustody.decrypt(locked.refreshTokenEncrypted!); + + let refreshed; + try { + refreshed = await runtime.refreshAccessToken!({ + config: ctx.config, + refreshToken, + signal: AbortSignal.timeout(ctx.timeoutMs ?? MANAGED_CREDENTIAL_REFRESH_TIMEOUT_MS), + }); + } catch (error) { + if (error instanceof ProviderResponseError && error.oauthError === "invalid_grant") { + revocation = "provider_revoked"; + return { kind: "revoke", updatedAt }; + } + throw new ManagedCredentialRefreshFailed(provider); + } + + let refreshTokenEncrypted: string; + if (refreshed.refreshToken) { + refreshTokenEncrypted = ctx.tokenCustody.encrypt(refreshed.refreshToken); + } else if (providerCatalog[provider].rotatesRefreshTokens) { + revocation = "rotation_missing"; + return { kind: "revoke", updatedAt }; + } else { + refreshTokenEncrypted = locked.refreshTokenEncrypted!; + } + + const tokenExpiresAt = refreshed.expiresAt ?? locked.tokenExpiresAt; + return { + kind: "rotate", + accessTokenEncrypted: ctx.tokenCustody.encrypt(refreshed.accessToken), + refreshTokenEncrypted, + ...(tokenExpiresAt !== undefined ? { tokenExpiresAt } : {}), + scopes: refreshed.scope.length ? refreshed.scope : locked.scopes, + updatedAt, + }; + }; + + const committed = await ctx.store.updateLinkedIdentityCredential(snapshot.id, decide); + if (revocation) { + await ctx.store.createAuditEvent({ + accountId: snapshot.accountId, + eventType: "managed_credential_revoked", + eventPayloadJson: { provider, reason: revocation }, + createdAt: new Date(clock()).toISOString(), + }); + } + if (!committed) { + return { kind: "revoked", identity: withoutTokens(snapshot) }; + } + return { kind: committed.accessTokenEncrypted ? "current" : "revoked", identity: committed }; +} diff --git a/src/oauth.ts b/src/oauth.ts index 2d0020c..187df6c 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -61,6 +61,25 @@ export interface OAuthProviderRuntime { }): Promise; } +const oauthErrorCodes = [ + "invalid_request", + "invalid_client", + "invalid_grant", + "unauthorized_client", + "unsupported_grant_type", + "invalid_scope", +] as const; +export type OAuthErrorCode = (typeof oauthErrorCodes)[number]; + +function readOAuthErrorCode(text: string): OAuthErrorCode | undefined { + try { + const error = (JSON.parse(text) as { error?: unknown } | null)?.error; + return oauthErrorCodes.find((code) => code === error); + } catch { + return undefined; + } +} + /** * A provider answer Heimdall cannot use: a non-2xx status, or a body that is * not JSON. It carries the status and a code, never the body or the parser's @@ -71,12 +90,18 @@ export interface OAuthProviderRuntime { export class ProviderResponseError extends Error { readonly status: number; readonly code: "provider_status" | "provider_body_not_json"; + /** + * The RFC 6749 section 5.2 error code of a refusal, when the provider sent + * one from that closed set. The only value taken from a provider's body. + */ + readonly oauthError: OAuthErrorCode | undefined; - constructor(status: number, code: ProviderResponseError["code"], failure: string) { + constructor(status: number, code: ProviderResponseError["code"], failure: string, oauthError?: OAuthErrorCode) { super(`${failure} (status ${status}, ${code}).`); this.name = "ProviderResponseError"; this.status = status; this.code = code; + this.oauthError = oauthError; } } @@ -84,7 +109,7 @@ export class ProviderResponseError extends Error { async function readProviderJson(response: Response, failure: string): Promise { const text = await response.text(); if (!response.ok) { - throw new ProviderResponseError(response.status, "provider_status", failure); + throw new ProviderResponseError(response.status, "provider_status", failure, readOAuthErrorCode(text)); } if (!text) { return {} as T; diff --git a/src/store/in-memory.ts b/src/store/in-memory.ts index 5583ac4..0dea560 100644 --- a/src/store/in-memory.ts +++ b/src/store/in-memory.ts @@ -6,6 +6,7 @@ import { type CreateAuthCompletionInput, type CreateAuditEventInput, type CreateCapabilityGrantInput, + type CredentialDecision, type CreateEntitlementSnapshotInput, type CreateSessionInput, type CreatePrivateCommandReceiptInput, @@ -37,6 +38,8 @@ function isGrantActive(grant: StoredCapabilityGrant, appSlug: AppSlug, at: strin } export class InMemoryStore implements HeimdallStore { + /** Tail of each linked identity's credential-decision chain; an entry is removed when it drains. */ + private readonly credentialChains = new Map>(); private readonly accounts = new Map(); private readonly linkedIdentities = new Map(); private readonly grants = new Map(); @@ -174,6 +177,45 @@ export class InMemoryStore implements HeimdallStore { return clone(linkedIdentity); } + async updateLinkedIdentityCredential( + id: string, + decide: (locked: StoredLinkedIdentity) => Promise, + ): Promise { + const previous = this.credentialChains.get(id) ?? Promise.resolve(); + const run = previous.then(async () => { + const current = this.linkedIdentities.get(id); + if (!current) { + return null; + } + const decision = await decide(clone(current)); + if (decision.kind === "keep") { + return clone(current); + } + const next: StoredLinkedIdentity = { ...current, updatedAt: decision.updatedAt }; + delete next.accessTokenEncrypted; + delete next.refreshTokenEncrypted; + delete next.tokenExpiresAt; + if (decision.kind === "rotate") { + next.accessTokenEncrypted = decision.accessTokenEncrypted; + next.refreshTokenEncrypted = decision.refreshTokenEncrypted; + if (decision.tokenExpiresAt !== undefined) { + next.tokenExpiresAt = decision.tokenExpiresAt; + } + next.scopes = [...decision.scopes]; + } + this.linkedIdentities.set(id, clone(next)); + return clone(next); + }); + const tail = run.catch(() => undefined); + this.credentialChains.set(id, tail); + void tail.then(() => { + if (this.credentialChains.get(id) === tail) { + this.credentialChains.delete(id); + } + }); + return run; + } + async listLinkedIdentitiesForAccount(accountId: string): Promise { return [...this.linkedIdentities.values()] .filter((identity) => identity.accountId === accountId) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 64f013d..f138f06 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -8,6 +8,7 @@ import { type CreateAuthCompletionInput, type CreateAuditEventInput, type CreateCapabilityGrantInput, + type CredentialDecision, type CreateEntitlementSnapshotInput, type CreateSessionInput, type CreatePrivateCommandReceiptInput, @@ -546,7 +547,7 @@ export async function checkSchemaRequirements(pool: Pick, require } export class PostgresStore implements HeimdallStore { - constructor(private readonly pool: Pick) {} + constructor(private readonly pool: Pick) {} async ensureSchema(): Promise { await this.pool.query(CREATE_SCHEMA_SQL); @@ -722,6 +723,60 @@ export class PostgresStore implements HeimdallStore { return mapLinkedIdentityRow(expectRow(result.rows[0], "upsertLinkedIdentity")); } + async updateLinkedIdentityCredential( + id: string, + decide: (locked: StoredLinkedIdentity) => Promise, + ): Promise { + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + try { + const locked = await client.query( + "SELECT * FROM linked_identities WHERE id = $1 FOR UPDATE", + [id] + ); + if (!locked.rowCount) { + await client.query("COMMIT"); + return null; + } + const row = mapLinkedIdentityRow(expectRow(locked.rows[0], "updateLinkedIdentityCredential")); + const decision = await decide(row); + let committed = row; + if (decision.kind !== "keep") { + const rotated = decision.kind === "rotate" ? decision : undefined; + const written = await client.query( + ` + UPDATE linked_identities + SET access_token_encrypted = $2, + refresh_token_encrypted = $3, + token_expires_at = $4, + scopes = $5, + updated_at = $6 + WHERE id = $1 + RETURNING * + `, + [ + id, + nullable(rotated?.accessTokenEncrypted), + nullable(rotated?.refreshTokenEncrypted), + nullable(rotated?.tokenExpiresAt), + (rotated?.scopes ?? row.scopes).join(" "), + decision.updatedAt, + ] + ); + committed = mapLinkedIdentityRow(expectRow(written.rows[0], "updateLinkedIdentityCredential")); + } + await client.query("COMMIT"); + return committed; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } finally { + client.release(); + } + } + async listLinkedIdentitiesForAccount(accountId: string): Promise { const result = await this.pool.query( ` diff --git a/src/store/types.ts b/src/store/types.ts index d1fabcf..0145626 100644 --- a/src/store/types.ts +++ b/src/store/types.ts @@ -56,6 +56,23 @@ export interface UpsertLinkedIdentityInput { updatedAt: string; } +/** + * What the owner of a refresh decides after reading the row under its lock. + * `rotate` and `revoke` write token columns, scopes and updated_at only; they + * never touch account, profile or identity fields. + */ +export type CredentialDecision = + | { kind: "keep" } + | { + kind: "rotate"; + accessTokenEncrypted: string; + refreshTokenEncrypted: string; + tokenExpiresAt?: string; + scopes: string[]; + updatedAt: string; + } + | { kind: "revoke"; updatedAt: string }; + export interface StoredSession { id: string; accountId: string; @@ -213,6 +230,17 @@ export interface HeimdallStore { findStoredLinkedIdentityForAccount(accountId: string, provider: Provider): Promise; listStoredLinkedIdentitiesForAccount(accountId: string): Promise; upsertLinkedIdentity(input: UpsertLinkedIdentityInput): Promise; + /** + * Serialise decisions about one row's credential. `decide` receives the row + * as read under the lock (never a caller's snapshot) and the decision is + * committed before this resolves, so a rotated pair is persisted before any + * caller sees the new access token. Returns the committed row, or null when + * no row has that id. A throw from `decide` writes nothing. + */ + updateLinkedIdentityCredential( + id: string, + decide: (locked: StoredLinkedIdentity) => Promise, + ): Promise; listLinkedIdentitiesForAccount(accountId: string): Promise; createCapabilityGrant(input: CreateCapabilityGrantInput): Promise; listActiveGrants(accountId: string, appSlug: AppSlug, at: string): Promise; diff --git a/tests/app.test.ts b/tests/app.test.ts index 7f9a72c..d938b9c 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -5,7 +5,7 @@ import { buildApp, errorIdentity } from "../src/app.js"; import { deliverBackendHandoff, type BackendHandoffPayload } from "../src/backend-handoff.js"; import { type HeimdallConfig, loadConfig } from "../src/config.js"; import { entitlementFacts, identityFacts } from "../src/facts.js"; -import { createOAuthRuntimeRegistry, type OAuthProviderRuntime } from "../src/oauth.js"; +import { createOAuthRuntimeRegistry, ProviderResponseError, type OAuthProviderRuntime } from "../src/oauth.js"; import { verifyJwt } from "../src/signing.js"; import { InMemoryStore } from "../src/store/index.js"; @@ -1238,6 +1238,117 @@ describe("Heimdall service", () => { expect(credentialResponse.body).not.toContain("rotated-twitch-refresh-token"); }); + it("revocation-is-typed: a revoked identity contributes no entitlement at session refresh", async () => { + const store = new InMemoryStore(); + let evaluations = 0; + const base = createMockDiscordRuntime(); + const app = await buildApp({ + config: createTestConfig(), + store, + oauthRuntimes: { + discord: { + ...base, + async exchangeAuthorizationCode() { + return { + accessToken: "expired-discord-access-token", + refreshToken: "discord-refresh-token", + tokenType: "Bearer", + scope: ["identify"], + expiresAt: "2026-04-26T13:00:00.000Z", + raw: {}, + }; + }, + async refreshAccessToken() { + throw new ProviderResponseError(400, "provider_status", "Discord token refresh failed", "invalid_grant"); + }, + async evaluateEntitlements(options) { + evaluations += 1; + return base.evaluateEntitlements(options); + }, + }, + }, + }); + apps.push(app); + + const stateToken = await startDiscordSignIn(app); + const callbackResponse = await app.inject({ + method: "GET", + url: `/v1/oauth/discord/callback?code=test-code&state=${encodeURIComponent(stateToken)}`, + }); + expect(callbackResponse.statusCode).toBe(201); + const evaluationsAtSignIn = evaluations; + + const refreshResponse = await app.inject({ + method: "POST", + url: "/v1/apps/repixelizer/sessions/refresh", + headers: { "x-heimdall-app-secret": "repixelizer-secret" }, + payload: { + refreshToken: callbackResponse.json().refreshToken as string, + entitlementPolicies: [{ kind: "discord_role_access", guildId: "gamecult-guild", allowedRoleIds: ["role-repixelizer"] }], + }, + }); + + expect(refreshResponse.statusCode).toBe(201); + expect(evaluations).toBe(evaluationsAtSignIn); + const accountId = callbackResponse.json().account.id as string; + const row = await store.findStoredLinkedIdentityForAccount(accountId, "discord"); + expect(row?.accessTokenEncrypted).toBeUndefined(); + expect(row?.refreshTokenEncrypted).toBeUndefined(); + }); + + it("revocation-is-typed: a revoked Patreon identity is not linked for Bifrost sync and Bifrost is not called", async () => { + const baseConfig = createTestConfig(); + const store = new InMemoryStore(); + const app = await buildApp({ + config: { + ...baseConfig, + bifrostPatronSupportEndpoint: "https://bifrost.gamecult.org/heimdall/patron-support/events", + bifrostPatronSupportSecret: "bifrost-support-secret", + }, + store, + oauthRuntimes: { + patreon: { + ...createOAuthRuntimeRegistry().patreon, + async refreshAccessToken() { + throw new ProviderResponseError(400, "provider_status", "Patreon token refresh failed", "invalid_grant"); + }, + }, + }, + }); + apps.push(app); + const context = getHeimdallContext(app); + const now = "2026-06-09T17:00:00.000Z"; + await store.createAccount({ id: "acct-p", createdAt: now, lastSeenAt: now }); + await store.upsertLinkedIdentity({ + accountId: "acct-p", + provider: "patreon", + providerUserId: "patreon-user-456", + accessTokenEncrypted: context.tokenCustody.encrypt("patreon-access-token"), + refreshTokenEncrypted: context.tokenCustody.encrypt("patreon-refresh-token"), + tokenExpiresAt: "2026-06-09T17:01:00.000Z", + scopes: [], + profileJson: {}, + createdAt: now, + updatedAt: now, + }); + const urls: string[] = []; + globalThis.fetch = async (input) => { + urls.push(String(input)); + return new Response("unexpected", { status: 500 }); + }; + + const response = await app.inject({ + method: "POST", + url: "/v1/apps/bifrost/patron-support/sync", + headers: { "x-heimdall-app-secret": "bifrost-secret" }, + payload: { accountId: "acct-p", requiredTierTitle: "Inner Sanctum", supportedAtUtc: now }, + }); + + expect(response.statusCode).toBe(404); + expect(response.json()).toEqual({ error: "patreon_identity_not_linked" }); + expect(urls).toEqual([]); + }); + it("no longer honors a global shared secret across apps", async () => { // A per-app identity model with a global fallback is a two-app shared // secret in practice (R6, fork 2). GC_ACCESS_APP_SHARED_SECRET alone must diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts new file mode 100644 index 0000000..b0dc2a2 --- /dev/null +++ b/tests/managed-credential.test.ts @@ -0,0 +1,517 @@ +import { randomUUID } from "node:crypto"; +import { Pool } from "pg"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { buildApp } from "../src/app.js"; +import { type HeimdallConfig } from "../src/config.js"; +import { type Provider } from "../src/contracts.js"; +import { createTokenCustody } from "../src/custody.js"; +import { + MANAGED_CREDENTIAL_REFRESH_SKEW_MS, + ManagedCredentialRefreshFailed, + refreshManagedCredential, +} from "../src/managed-credential.js"; +import { createOAuthRuntimeRegistry } from "../src/oauth.js"; +import { providerCatalog } from "../src/providers.js"; +import { InMemoryStore, PostgresStore, type HeimdallStore } from "../src/store/index.js"; +import { type StoredLinkedIdentity } from "../src/store/types.js"; + +const originalFetch = globalThis.fetch; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function testConfig(): HeimdallConfig { + const clients = Object.fromEntries( + ["discord", "patreon", "github", "twitch", "youtube", "spotify", "joystick"].map((p) => [ + p, + { clientId: `${p}-client`, clientSecret: `${p}-secret` }, + ]) + ); + return { + serviceName: "heimdall", + host: "127.0.0.1", + port: 4100, + workspaceRoot: "E:/Projects/Heimdall", + dataRoot: "E:/Projects/Heimdall/.heimdall-data", + cultCachePath: "E:/Projects/Heimdall/.heimdall-data/cultcache/heimdall.service.cc", + publicBaseUrl: "https://heimdall.gamecult.org", + issuer: "https://heimdall.gamecult.org", + daemonId: "yggdrasil-heimdall", + idunnRudpHealth: undefined, + idunnHealthContract: "heimdall.cultnet-rudp-provider-health", + providerHealthIdentityPath: "E:/Projects/Heimdall/.heimdall-data/provider-health-identity.cc", + sessionTtlSeconds: 3600, + refreshTtlSeconds: 60 * 60 * 24 * 30, + stateTtlSeconds: 600, + completionTtlSeconds: 300, + bootstrapSigningPrivateKeyOnMissing: false, + tokenEncryptionKeyBase64: Buffer.alloc(32, 7).toString("base64"), + appSharedSecrets: { streampixels: "streampixels-secret", bifrost: "bifrost-secret" }, + appBackendCallbacks: {}, + storage: { backend: "memory", applySchemaOnStartup: true }, + bifrostPatronSupportEndpoint: "https://bifrost.gamecult.org/heimdall/patron-support/events", + bifrostPatronSupportSecret: "bifrost-support-secret", + providers: clients, + } as unknown as HeimdallConfig; +} + +type Mode = "normal" | "invalid_grant" | "invalid_client" | "server_error" | "not_json" | "no_refresh_token" | "hang"; + +/** + * A provider token endpoint with single-use refresh tokens. Presenting a + * consumed refresh token revokes the whole installation, as Joystick does: + * every later refresh answers 400 invalid_grant. + */ +function installTokenEndpoint(provider: Provider, firstRefreshToken: string) { + const state = { + calls: 0, + mode: "normal" as Mode, + live: firstRefreshToken, + issued: 0, + installationRevoked: false, + otherCalls: [] as string[], + }; + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + const url = String(input); + if (url !== providerCatalog[provider].tokenEndpoint) { + state.otherCalls.push(url); + return new Response("unexpected", { status: 500 }); + } + state.calls += 1; + const presented = (init!.body as URLSearchParams).get("refresh_token"); + await sleep(25); + switch (state.mode) { + case "hang": + return new Promise((_resolve, reject) => { + init!.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); + }); + case "invalid_grant": + return new Response(JSON.stringify({ error: "invalid_grant", error_description: "CANARY" }), { status: 400 }); + case "invalid_client": + return new Response(JSON.stringify({ error: "invalid_client", error_description: "CANARY" }), { status: 401 }); + case "server_error": + return new Response("CANARY upstream exploded", { status: 500 }); + case "not_json": + return new Response("CANARY ", { status: 200 }); + case "no_refresh_token": + return new Response(JSON.stringify({ access_token: "access-norefresh", token_type: "Bearer", expires_in: 3600 }), { + status: 200, + }); + case "normal": { + if (state.installationRevoked || presented !== state.live) { + state.installationRevoked = true; + return new Response(JSON.stringify({ error: "invalid_grant", error_description: "reused CANARY" }), { status: 400 }); + } + state.issued += 1; + state.live = `refresh-${state.issued}`; + return new Response( + JSON.stringify({ + access_token: `access-${state.issued}`, + refresh_token: state.live, + token_type: "Bearer", + expires_in: 3600, + }), + { status: 200 } + ); + } + } + }) as typeof fetch; + return state; +} + +interface Rig { + stores: HeimdallStore[]; + apps: Awaited>[]; + accountId: string; + identityId: string; + custody: ReturnType; + config: HeimdallConfig; + provider: Provider; + row(): Promise; + audit(): Promise>; + resolve(appIndex?: number): Promise<{ statusCode: number; json: Record; body: string }>; + close(): Promise; +} + +const adminUrl = process.env.HEIMDALL_TEST_PG_ADMIN_URL; +const rigKinds = [ + { kind: "memory", live: false }, + { kind: "postgres one app", live: true }, + { kind: "postgres two apps", live: true }, +] as const; +// Providers whose refresh goes through the real shared runtime; rotation differs by descriptor. +const refreshProviders: Provider[] = ["twitch"]; + +for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`managed credential refresh, ${kind}`, () => { + const admin = live && adminUrl ? new Pool({ connectionString: adminUrl, max: 2 }) : undefined; + const suffix = randomUUID().replace(/-/g, "").slice(0, 12); + const template = `heimdall_mc_template_${suffix}`; + const cleanup: Array<() => Promise> = []; + + function urlFor(database: string): string { + const url = new URL(adminUrl!); + url.pathname = `/${database}`; + return url.toString(); + } + + beforeAll(async () => { + if (!live) return; + await admin!.query(`CREATE DATABASE ${template}`); + const pool = new Pool({ connectionString: urlFor(template), max: 1 }); + await new PostgresStore(pool).ensureSchema(); + await pool.end(); + }, 120_000); + + afterEach(async () => { + globalThis.fetch = originalFetch; + for (const step of cleanup.splice(0).reverse()) await step(); + }); + + afterAll(async () => { + if (!admin) return; + await admin.query(`DROP DATABASE IF EXISTS ${template}`); + await admin.end(); + }, 120_000); + + async function makeRig( + provider: Provider, + options: { expiresInMs?: number; refreshToken?: string | null; accessToken?: string | null } = {} + ): Promise { + const config = testConfig(); + const custody = createTokenCustody(config); + const stores: HeimdallStore[] = []; + let adminForDb: Pool | undefined; + if (!live) { + stores.push(new InMemoryStore()); + } else { + const name = `heimdall_mc_${randomUUID().replace(/-/g, "")}`; + await admin!.query(`CREATE DATABASE ${name} TEMPLATE ${template}`); + const count = kind === "postgres two apps" ? 2 : 1; + for (let i = 0; i < count; i += 1) { + const pool = new Pool({ connectionString: urlFor(name), max: 12 }); + pool.on("error", () => undefined); + stores.push(new PostgresStore(pool)); + } + adminForDb = new Pool({ connectionString: urlFor(name), max: 1 }); + adminForDb.on("error", () => undefined); + cleanup.push(async () => { + await adminForDb!.end(); + await admin!.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); + }); + } + + const now = new Date().toISOString(); + const account = await stores[0]!.createAccount({ createdAt: now, lastSeenAt: now }); + const expiresInMs = options.expiresInMs ?? 60_000; + const input = { + accountId: account.id, + provider, + providerUserId: "channel-1", + username: "deru", + displayName: "Deru", + scopes: ["identity:read"], + profileJson: { keep: "me" }, + tokenExpiresAt: new Date(Date.now() + expiresInMs).toISOString(), + createdAt: now, + updatedAt: now, + }; + const seeded = await stores[0]!.upsertLinkedIdentity({ + ...input, + ...(options.accessToken === null ? {} : { accessTokenEncrypted: custody.encrypt(options.accessToken ?? "access-0") }), + ...(options.refreshToken === null ? {} : { refreshTokenEncrypted: custody.encrypt(options.refreshToken ?? "refresh-0") }), + }); + + const apps: Rig["apps"] = []; + for (const store of stores) { + apps.push(await buildApp({ config, store, tokenCustody: custody, oauthRuntimes: createOAuthRuntimeRegistry() })); + } + cleanup.push(async () => { + for (const app of apps) await app.close().catch(() => undefined); + }); + + return { + stores, + apps, + accountId: account.id, + identityId: seeded.id, + custody, + config, + provider, + async row() { + const row = await stores[0]!.findStoredLinkedIdentityForAccount(account.id, provider); + return row!; + }, + async audit() { + if (!live) { + const events = (stores[0] as unknown as { auditEvents: Map }).auditEvents; + return [...events.values()].map((e) => ({ eventType: e.eventType, payload: e.eventPayloadJson })); + } + const result = await adminForDb!.query<{ event_type: string; event_payload_json: unknown }>( + "SELECT event_type, event_payload_json FROM audit_events ORDER BY created_at" + ); + return result.rows.map((r) => ({ eventType: r.event_type, payload: r.event_payload_json })); + }, + async resolve(appIndex = 0) { + const response = await apps[appIndex]!.inject({ + method: "POST", + url: "/v1/apps/streampixels/managed-credentials/resolve", + headers: { "x-heimdall-app-secret": "streampixels-secret" }, + payload: { accountId: account.id, provider }, + }); + return { statusCode: response.statusCode, json: response.json() as Record, body: response.body }; + }, + async close() {}, + }; + } + + const timeout = { timeout: 60_000 }; + + describe.each(refreshProviders)("%s", (provider) => { + it("refresh-once-per-token: 20 concurrent resolves present the refresh token once", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + + const answers = await Promise.all(Array.from({ length: 20 }, (_, i) => rig.resolve(i % rig.apps.length))); + + expect(endpoint.calls).toBe(1); + expect(answers.map((a) => a.statusCode)).toEqual(Array(20).fill(200)); + expect(new Set(answers.map((a) => a.json.accessToken))).toEqual(new Set(["access-1"])); + const row = await rig.row(); + expect(rig.custody.decrypt(row.accessTokenEncrypted!)).toBe("access-1"); + expect(rig.custody.decrypt(row.refreshTokenEncrypted!)).toBe("refresh-1"); + expect(await rig.audit()).toEqual([]); + }); + + it("rotated-pair-persisted-before-use: the stored pair is the one the caller was given", timeout, async () => { + const rig = await makeRig(provider); + installTokenEndpoint(provider, "refresh-0"); + const answer = await rig.resolve(); + const row = await rig.row(); + expect(rig.custody.decrypt(row.accessTokenEncrypted!)).toBe(answer.json.accessToken); + expect(Date.parse(row.tokenExpiresAt!)).toBeGreaterThan(Date.now() + MANAGED_CREDENTIAL_REFRESH_SKEW_MS); + }); + + it("outside the skew: no token call, the stored token is returned", timeout, async () => { + const rig = await makeRig(provider, { expiresInMs: 60 * 60 * 1000 }); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + const answer = await rig.resolve(); + expect(endpoint.calls).toBe(0); + expect(answer.statusCode).toBe(200); + expect(answer.json.accessToken).toBe("access-0"); + }); + + it("revocation-is-typed: invalid_grant clears the tokens, answers 410, audits once, and stops calling the provider", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "invalid_grant"; + + const first = await rig.resolve(); + expect(first.statusCode).toBe(410); + expect(first.json).toEqual({ error: "managed_credential_revoked", provider }); + const row = await rig.row(); + expect(row.accessTokenEncrypted).toBeUndefined(); + expect(row.refreshTokenEncrypted).toBeUndefined(); + expect(row.tokenExpiresAt).toBeUndefined(); + expect(row.username).toBe("deru"); + expect(row.profileJson).toEqual({ keep: "me" }); + expect(row.scopes).toEqual(["identity:read"]); + + const second = await rig.resolve(); + expect(second.statusCode).toBe(410); + expect(endpoint.calls).toBe(1); + + const events = await rig.audit(); + expect(events.filter((e) => e.eventType === "managed_credential_revoked")).toEqual([ + { eventType: "managed_credential_revoked", payload: { provider, reason: "provider_revoked" } }, + ]); + expect(first.body + second.body + JSON.stringify(events)).not.toContain("CANARY"); + }); + + it.each(["invalid_client", "server_error", "not_json", "hang"] as const)( + "transient failure (%s) keeps custody: 502, tokens byte-identical, nothing leaked", + timeout, + async (mode) => { + const rig = await makeRig(provider); + const before = await rig.row(); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = mode; + + let answer; + if (mode === "hang") { + // The route's own bound is 10 s; the primitive takes an injected one. + const failure = await refreshManagedCredential( + { config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 100 }, + before + ).then( + () => undefined, + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(ManagedCredentialRefreshFailed); + expect((failure as ManagedCredentialRefreshFailed).provider).toBe(provider); + expect(String(failure) + JSON.stringify(failure)).not.toContain("CANARY"); + } else { + answer = await rig.resolve(); + expect(answer.statusCode).toBe(502); + expect(answer.json).toEqual({ error: "managed_credential_refresh_failed", provider }); + expect(answer.body).not.toContain("CANARY"); + } + + const after = await rig.row(); + expect(after.accessTokenEncrypted).toBe(before.accessTokenEncrypted); + expect(after.refreshTokenEncrypted).toBe(before.refreshTokenEncrypted); + expect(after.tokenExpiresAt).toBe(before.tokenExpiresAt); + expect(await rig.audit()).toEqual([]); + } + ); + + it("bounded lock: a refresh that never answers is aborted and the next resolve takes the lock", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "hang"; + const ctx = { config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 100 }; + const started = Date.now(); + await expect(refreshManagedCredential(ctx, await rig.row())).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect(Date.now() - started).toBeLessThan(5_000); + + endpoint.mode = "normal"; + const outcome = await refreshManagedCredential(ctx, await rig.row()); + expect(outcome.kind).toBe("current"); + expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1"); + }); + + it("a snapshot taken before another caller's refresh does not decide: the locked row does", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + const stale = await rig.row(); + await rig.resolve(); + expect(endpoint.calls).toBe(1); + const outcome = await refreshManagedCredential( + { config: rig.config, store: rig.stores[0]!, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry() }, + stale + ); + expect(endpoint.calls).toBe(1); + expect(outcome.kind).toBe("current"); + expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1"); + }); + + it("a response without a refresh token keeps the old one for a non-rotating provider", timeout, async () => { + expect(providerCatalog[provider].rotatesRefreshTokens).toBe(false); + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "no_refresh_token"; + const answer = await rig.resolve(); + expect(answer.statusCode).toBe(200); + expect(answer.json.accessToken).toBe("access-norefresh"); + expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0"); + }); + + it("a row with no access token resolves as revoked (410), a missing row as not found (404)", timeout, async () => { + const rig = await makeRig(provider, { accessToken: null, refreshToken: null }); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + const revoked = await rig.resolve(); + expect(revoked.statusCode).toBe(410); + expect(revoked.json).toEqual({ error: "managed_credential_revoked", provider }); + expect(endpoint.calls).toBe(0); + + const missing = await rig.apps[0]!.inject({ + method: "POST", + url: "/v1/apps/streampixels/managed-credentials/resolve", + headers: { "x-heimdall-app-secret": "streampixels-secret" }, + payload: { accountId: "no-such-account", provider }, + }); + expect(missing.statusCode).toBe(404); + expect(missing.json()).toEqual({ error: "managed_credential_not_found" }); + }); + }); + + describe("updateLinkedIdentityCredential", () => { + it("returns null for a missing id and does not call decide", async () => { + const rig = await makeRig("twitch"); + let called = false; + const result = await rig.stores[0]!.updateLinkedIdentityCredential(randomUUID(), async () => { + called = true; + return { kind: "keep" }; + }); + expect(result).toBeNull(); + expect(called).toBe(false); + }); + + it("hands decide the committed row, and a throw writes nothing", async () => { + const rig = await makeRig("twitch"); + const before = await rig.row(); + let seen: StoredLinkedIdentity | undefined; + await expect( + rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async (locked) => { + seen = locked; + throw new Error("boom"); + }) + ).rejects.toThrow("boom"); + expect(seen).toEqual(before); + expect(await rig.row()).toEqual(before); + }); + + it("rotate writes token columns, scopes and updated_at and nothing else; keep writes nothing", async () => { + const rig = await makeRig("twitch"); + const before = await rig.row(); + const kept = await rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async () => ({ kind: "keep" })); + expect(kept).toEqual(before); + + const rotated = await rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async () => ({ + kind: "rotate", + accessTokenEncrypted: "A2", + refreshTokenEncrypted: "R2", + tokenExpiresAt: "2999-01-01T00:00:00.000Z", + scopes: ["a", "b"], + updatedAt: "2999-01-02T00:00:00.000Z", + })); + expect(rotated).toEqual({ + ...before, + accessTokenEncrypted: "A2", + refreshTokenEncrypted: "R2", + tokenExpiresAt: "2999-01-01T00:00:00.000Z", + scopes: ["a", "b"], + updatedAt: "2999-01-02T00:00:00.000Z", + }); + expect(await rig.row()).toEqual(rotated); + + const revoked = await rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async () => ({ + kind: "revoke", + updatedAt: "2999-01-03T00:00:00.000Z", + })); + const { accessTokenEncrypted: _a, refreshTokenEncrypted: _r, tokenExpiresAt: _e, ...rest } = rotated!; + expect(revoked).toEqual({ ...rest, updatedAt: "2999-01-03T00:00:00.000Z" }); + expect(await rig.row()).toEqual(revoked); + }); + + it("serialises deciders on one row: each sees the previous commit", timeout, async () => { + const rig = await makeRig("twitch"); + const order: string[] = []; + await Promise.all( + [1, 2, 3, 4, 5].map((n) => + rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async (locked) => { + order.push(`${n}:${locked.accessTokenEncrypted}`); + await sleep(20); + return { + kind: "rotate", + accessTokenEncrypted: `A${n}`, + refreshTokenEncrypted: `R${n}`, + scopes: locked.scopes, + updatedAt: new Date().toISOString(), + }; + }) + ) + ); + const seenValues = order.map((entry) => entry.split(":")[1]); + // Each decider saw the value the one before it wrote, so their reads form a chain. + expect(new Set(seenValues).size).toBe(5); + const wrote = order.map((entry) => `A${entry.split(":")[0]}`); + for (let i = 1; i < seenValues.length; i += 1) expect(seenValues[i]).toBe(wrote[i - 1]); + }); + }); +}); + +describe("live Postgres availability", () => { + it.skipIf(!!adminUrl)("is present whenever the job requires it", () => { + // Local workstation runs skip the live suites; the verification job sets + // HEIMDALL_REQUIRE_LIVE_PG=1 and fails here when they would have been skipped. + expect(process.env.HEIMDALL_REQUIRE_LIVE_PG).toBeUndefined(); + }); +}); diff --git a/tests/provider-token-request.test.ts b/tests/provider-token-request.test.ts index 2d13a30..ff13d72 100644 --- a/tests/provider-token-request.test.ts +++ b/tests/provider-token-request.test.ts @@ -1,3 +1,4 @@ +import { inspect } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { type HeimdallConfig } from "../src/config.js"; import { providers, type Provider } from "../src/contracts.js"; @@ -114,3 +115,33 @@ describe("requestProviderTokens follows the provider descriptor", () => { ).rejects.toThrow("Patreon OAuth is not fully configured."); }); }); + +describe("oauthError is a closed set", () => { + const refresh = { grant_type: "refresh_token", refresh_token: "r" } as const; + async function failure(body: string, status = 400) { + capture(() => new Response(body, { status })); + return requestProviderTokens("twitch", configFor(), refresh).then( + () => { + throw new Error("expected a refusal"); + }, + (error: unknown) => error as Error & { oauthError?: string } + ); + } + + it("takes invalid_grant from the body and carries nothing else of it", async () => { + const error = await failure(JSON.stringify({ error: "invalid_grant", error_description: "CANARY" })); + expect(error.oauthError).toBe("invalid_grant"); + expect(`${error.message} ${error.stack} ${inspect(error, { depth: 5, showHidden: true })} ${JSON.stringify(error)}`).not.toContain("CANARY"); + }); + + it.each([ + ["an error outside the RFC set", JSON.stringify({ error: "CANARY_unknown" })], + ["a non-string error", JSON.stringify({ error: { code: "invalid_grant" } })], + ["a body that is not JSON", "CANARY "], + ["JSON null", "null"], + ])("leaves oauthError undefined for %s", async (_name, body) => { + const error = await failure(body); + expect(error.oauthError).toBeUndefined(); + expect(`${error.message} ${inspect(error, { depth: 5, showHidden: true })}`).not.toContain("CANARY"); + }); +}); diff --git a/tests/store.test.ts b/tests/store.test.ts index 8eeaeb1..c5f6445 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -326,7 +326,8 @@ async function exerciseStore(store: PostgresStore): Promise> { createdAt: now, updatedAt: now, }; - await s.upsertLinkedIdentity(identity); + const linked = await s.upsertLinkedIdentity(identity); + await s.updateLinkedIdentityCredential(linked.id, async () => ({ kind: "keep" })); await s.upsertLinkedIdentity({ ...identity, username: "again" }); await s.findAccountByLinkedIdentity("discord", "u1"); await s.findStoredLinkedIdentity("discord", "u1"); From 1c38d12050c295d2739d39d0aa8fa49b9075fba3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:13:40 +0200 Subject: [PATCH 49/85] Add Joystick as a confidential, rotating managed-connection provider Joystick joins the catalog (Basic client auth, rotatesRefreshTokens), resolves identity by channel id, and is a streampixels managed-connection provider only: it is not an identity provider for the profile (ruling signin-waits-for-public-bot). Co-Authored-By: Claude Opus 5.5 --- ...heimdall.auth_begin_command.v1.schema.json | 2 +- src/app-profiles.ts | 4 +- src/contracts.ts | 2 +- src/oauth.ts | 49 ++++++++ src/providers.ts | 12 ++ tests/access-plugin.test.ts | 2 +- tests/app.test.ts | 107 +++++++++++++++++- tests/caller-identity-attack-soul-r21.test.ts | 1 + tests/caller-identity-attack-soul.test.ts | 1 + tests/caller-identity-soul.test.ts | 1 + tests/managed-credential.test.ts | 45 ++++++-- tests/private-command-diagnostics.test.ts | 2 +- tests/security.test.ts | 1 + tests/verse-witness.test.ts | 1 + 14 files changed, 212 insertions(+), 18 deletions(-) diff --git a/schemas/heimdall.auth_begin_command.v1.schema.json b/schemas/heimdall.auth_begin_command.v1.schema.json index 81fdaf8..ca22ccf 100644 --- a/schemas/heimdall.auth_begin_command.v1.schema.json +++ b/schemas/heimdall.auth_begin_command.v1.schema.json @@ -4,7 +4,7 @@ "type": "object", "required": ["provider", "mode", "returnTo", "entitlementPolicy"], "properties": { - "provider": { "enum": ["discord", "patreon", "github", "twitch", "youtube", "spotify"] }, + "provider": { "enum": ["discord", "patreon", "github", "twitch", "youtube", "spotify", "joystick"] }, "mode": { "enum": ["sign_in", "link", "connect"] }, "returnTo": { "type": "string", "format": "uri" }, "entitlementPolicy": { diff --git a/src/app-profiles.ts b/src/app-profiles.ts index bede115..88466d9 100644 --- a/src/app-profiles.ts +++ b/src/app-profiles.ts @@ -99,10 +99,10 @@ const repixelizerProfile: AppProfile = { const streampixelsProfile: AppProfile = { slug: "streampixels", displayName: "StreamPixels", - profileVersion: "2026-04-26", + profileVersion: "2026-10-19", identityProviders: ["twitch", "youtube"], entitlementSources: [], - managedConnectionProviders: ["twitch", "youtube"], + managedConnectionProviders: ["twitch", "youtube", "joystick"], allowedReturnOrigins: ["https://streampixels.gamecult.org"], capabilities: [ { diff --git a/src/contracts.ts b/src/contracts.ts index 743bfdc..d6cab2a 100644 --- a/src/contracts.ts +++ b/src/contracts.ts @@ -1,4 +1,4 @@ -export const providers = ["discord", "patreon", "github", "twitch", "youtube", "spotify"] as const; +export const providers = ["discord", "patreon", "github", "twitch", "youtube", "spotify", "joystick"] as const; export type Provider = (typeof providers)[number]; diff --git a/src/oauth.ts b/src/oauth.ts index 187df6c..c46976e 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -513,6 +513,45 @@ async function resolveSpotifyIdentity(options: { accessToken: string }): Promise return identity; } +async function resolveJoystickIdentity(options: { accessToken: string }): Promise { + const profile = await fetchJson<{ + channel_id?: string; + username?: string; + nickname?: string; + photo_url?: string; + }>( + "https://api.joystick.tv/api/v1/me/identity", + { + headers: { + Authorization: `Bearer ${options.accessToken}`, + }, + }, + "Joystick identity lookup failed" + ); + + if (!profile.channel_id) { + throw new Error("Joystick identity response did not include a channel id."); + } + + const identity: ResolvedIdentity = { + provider: "joystick", + providerUserId: profile.channel_id, + profile: profile as unknown as Record, + }; + const displayName = profile.nickname ?? profile.username; + if (profile.username) { + identity.username = profile.username; + } + if (displayName) { + identity.displayName = displayName; + } + if (profile.photo_url) { + identity.avatarUrl = profile.photo_url; + } + + return identity; +} + async function evaluateDiscordEntitlements(options: { config: HeimdallConfig; callback: OAuthCallbackContext; @@ -860,6 +899,15 @@ const spotifyRuntime: OAuthProviderRuntime = { }, }; +const joystickRuntime: OAuthProviderRuntime = { + exchangeAuthorizationCode: sharedRuntimeExchange("joystick"), + refreshAccessToken: sharedRuntimeRefresh("joystick"), + resolveIdentity: resolveJoystickIdentity, + async evaluateEntitlements() { + return { facts: [], snapshots: [] }; + }, +}; + const notImplementedRuntime: OAuthProviderRuntime = { async exchangeAuthorizationCode() { throw new Error("Provider callback exchange is not implemented yet."); @@ -882,5 +930,6 @@ export function createOAuthRuntimeRegistry(): OAuthRuntimeRegistry { twitch: twitchRuntime, youtube: youtubeRuntime, spotify: spotifyRuntime, + joystick: joystickRuntime, }; } diff --git a/src/providers.ts b/src/providers.ts index 2998e4d..9b139ab 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -105,6 +105,18 @@ export const providerCatalog: Record = { clientIdEnv: "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID", clientSecretEnv: "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE", }, + joystick: { + key: "joystick", + displayName: "Joystick", + authorizationEndpoint: "https://joystick.tv/api/oauth/authorize", + tokenEndpoint: "https://api.joystick.tv/api/oauth/token", + tokenEndpointAuthMethod: "client_secret_basic", + rotatesRefreshTokens: true, + defaultScopes: ["identity:read", "stream:read", "chat:read", "followers:read", "subscribers:read"], + roles: ["identity", "managed_credential"], + clientIdEnv: "GC_ACCESS_PROVIDER_JOYSTICK_CLIENT_ID", + clientSecretEnv: "GC_ACCESS_PROVIDER_JOYSTICK_CLIENT_SECRET_FILE", + }, }; export function buildAuthorizationUrl(options: { diff --git a/tests/access-plugin.test.ts b/tests/access-plugin.test.ts index b5027d0..4789bf8 100644 --- a/tests/access-plugin.test.ts +++ b/tests/access-plugin.test.ts @@ -291,7 +291,7 @@ function testConfig(): HeimdallConfig { storage: { backend: "memory", applySchemaOnStartup: true }, providers: { discord: { clientId: "discord-client", clientSecret: "discord-secret" }, - patreon: {}, github: {}, twitch: {}, youtube: {}, spotify: {}, + patreon: {}, github: {}, twitch: {}, youtube: {}, spotify: {}, joystick: {}, }, }; } diff --git a/tests/app.test.ts b/tests/app.test.ts index d938b9c..b2960fc 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -49,6 +49,7 @@ function createTestConfig(): HeimdallConfig { twitch: { clientId: "twitch-client", clientSecret: "twitch-secret" }, youtube: { clientId: "youtube-client", clientSecret: "youtube-secret" }, spotify: { clientId: "spotify-client", clientSecret: "spotify-secret" }, + joystick: { clientId: "joystick-client", clientSecret: "joystick-secret" }, }, }; } @@ -974,7 +975,8 @@ describe("Heimdall service", () => { expect(profileResponse.json()).toEqual( expect.objectContaining({ slug: "streampixels", - managedConnectionProviders: ["twitch", "youtube"], + identityProviders: ["twitch", "youtube"], + managedConnectionProviders: ["twitch", "youtube", "joystick"], }) ); }); @@ -1349,6 +1351,109 @@ describe("Heimdall service", () => { expect(urls).toEqual([]); }); + describe("joystick connect", () => { + async function connect(app: FastifyInstance) { + const start = await app.inject({ + method: "POST", + url: "/v1/oauth/joystick/start", + payload: { appSlug: "streampixels", mode: "connect", returnTo: "https://streampixels.gamecult.org/auth/connect" }, + }); + return start; + } + + it("starts a streampixels connect at joystick.tv with the five scopes and the callback redirect", async () => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + const start = await connect(app); + expect(start.statusCode).toBe(201); + const url = new URL(start.json().authorizationUrl as string); + expect(`${url.origin}${url.pathname}`).toBe("https://joystick.tv/api/oauth/authorize"); + expect(url.searchParams.get("client_id")).toBe("joystick-client"); + expect(url.searchParams.get("scope")).toBe("identity:read stream:read chat:read followers:read subscribers:read"); + expect(url.searchParams.get("redirect_uri")).toBe("https://heimdall.gamecult.org/v1/oauth/joystick/callback"); + }); + + function stubJoystick(identity: Record) { + const calls: Array<{ url: string; headers: Record; body?: URLSearchParams }> = []; + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + const url = String(input); + calls.push({ url, headers: (init?.headers ?? {}) as Record, body: init?.body as URLSearchParams | undefined }); + if (url === "https://api.joystick.tv/api/oauth/token") { + return new Response( + JSON.stringify({ access_token: "js-access", refresh_token: "js-refresh", token_type: "Bearer", expires_in: 864000 }), + { status: 200 } + ); + } + if (url === "https://api.joystick.tv/api/v1/me/identity") { + return new Response(JSON.stringify(identity), { status: 200 }); + } + return new Response("unexpected", { status: 500 }); + }) as typeof fetch; + return calls; + } + + it("exchanges the code with Basic client auth and links the identity by channel id", async () => { + const store = new InMemoryStore(); + const app = await buildApp({ config: createTestConfig(), store }); + apps.push(app); + const calls = stubJoystick({ channel_id: "chan-9", username: "deru", nickname: "Deru!", photo_url: "https://img/x.png" }); + const stateToken = (await connect(app)).json().stateToken as string; + + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=the-code&state=${encodeURIComponent(stateToken)}`, + }); + expect(callback.statusCode).toBeLessThan(300); + + const exchange = calls.find((c) => c.url === "https://api.joystick.tv/api/oauth/token")!; + expect(exchange.headers.Authorization).toBe(`Basic ${Buffer.from("joystick-client:joystick-secret").toString("base64")}`); + expect(exchange.body!.get("grant_type")).toBe("authorization_code"); + expect(exchange.body!.get("code")).toBe("the-code"); + expect(exchange.body!.get("redirect_uri")).toBe("https://heimdall.gamecult.org/v1/oauth/joystick/callback"); + expect(exchange.body!.has("client_secret")).toBe(false); + const identityCall = calls.find((c) => c.url === "https://api.joystick.tv/api/v1/me/identity")!; + expect(identityCall.headers.Authorization).toBe("Bearer js-access"); + + const accountId = callback.json().account.id as string; + const row = await store.findStoredLinkedIdentityForAccount(accountId, "joystick"); + expect(row).toMatchObject({ providerUserId: "chan-9", username: "deru", displayName: "Deru!" }); + expect(row?.accessTokenEncrypted).toEqual(expect.any(String)); + expect(row?.accessTokenEncrypted).not.toContain("js-access"); + expect(row?.refreshTokenEncrypted).toEqual(expect.any(String)); + const left = Date.parse(row!.tokenExpiresAt!) - Date.now(); + expect(left).toBeGreaterThan(863_000 * 1000); + expect(left).toBeLessThanOrEqual(864_000 * 1000); + }); + + it("falls back to the username for the display name", async () => { + const store = new InMemoryStore(); + const app = await buildApp({ config: createTestConfig(), store }); + apps.push(app); + stubJoystick({ channel_id: "chan-9", username: "deru" }); + const stateToken = (await connect(app)).json().stateToken as string; + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=c&state=${encodeURIComponent(stateToken)}`, + }); + const row = await store.findStoredLinkedIdentityForAccount(callback.json().account.id as string, "joystick"); + expect(row?.displayName).toBe("deru"); + }); + + it("fails the callback when the identity has no channel id", async () => { + const store = new InMemoryStore(); + const app = await buildApp({ config: createTestConfig(), store }); + apps.push(app); + stubJoystick({ username: "deru" }); + const stateToken = (await connect(app)).json().stateToken as string; + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=c&state=${encodeURIComponent(stateToken)}`, + }); + expect(callback.statusCode).toBeGreaterThanOrEqual(400); + expect(await store.findStoredLinkedIdentity("joystick", "undefined")).toBeNull(); + }); + }); + it("no longer honors a global shared secret across apps", async () => { // A per-app identity model with a global fallback is a two-app shared // secret in practice (R6, fork 2). GC_ACCESS_APP_SHARED_SECRET alone must diff --git a/tests/caller-identity-attack-soul-r21.test.ts b/tests/caller-identity-attack-soul-r21.test.ts index 4c3df32..de959a8 100644 --- a/tests/caller-identity-attack-soul-r21.test.ts +++ b/tests/caller-identity-attack-soul-r21.test.ts @@ -65,6 +65,7 @@ function testConfig(overrides: Partial = {}): HeimdallConfig { twitch: {}, youtube: {}, spotify: {}, + joystick: {}, }, ...overrides, }; diff --git a/tests/caller-identity-attack-soul.test.ts b/tests/caller-identity-attack-soul.test.ts index 9fb8e0a..8f77801 100644 --- a/tests/caller-identity-attack-soul.test.ts +++ b/tests/caller-identity-attack-soul.test.ts @@ -51,6 +51,7 @@ function testConfig(): HeimdallConfig { twitch: {}, youtube: {}, spotify: {}, + joystick: {}, }, }; } diff --git a/tests/caller-identity-soul.test.ts b/tests/caller-identity-soul.test.ts index 3ebc000..b5562a7 100644 --- a/tests/caller-identity-soul.test.ts +++ b/tests/caller-identity-soul.test.ts @@ -59,6 +59,7 @@ function testConfig(): HeimdallConfig { twitch: {}, youtube: {}, spotify: {}, + joystick: {}, }, }; } diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index b0dc2a2..427b2c8 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -138,7 +138,7 @@ const rigKinds = [ { kind: "postgres two apps", live: true }, ] as const; // Providers whose refresh goes through the real shared runtime; rotation differs by descriptor. -const refreshProviders: Provider[] = ["twitch"]; +const refreshProviders: Provider[] = ["twitch", "joystick"]; for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`managed credential refresh, ${kind}`, () => { const admin = live && adminUrl ? new Pool({ connectionString: adminUrl, max: 2 }) : undefined; @@ -392,16 +392,39 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1"); }); - it("a response without a refresh token keeps the old one for a non-rotating provider", timeout, async () => { - expect(providerCatalog[provider].rotatesRefreshTokens).toBe(false); - const rig = await makeRig(provider); - const endpoint = installTokenEndpoint(provider, "refresh-0"); - endpoint.mode = "no_refresh_token"; - const answer = await rig.resolve(); - expect(answer.statusCode).toBe(200); - expect(answer.json.accessToken).toBe("access-norefresh"); - expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0"); - }); + if (providerCatalog[provider].rotatesRefreshTokens) { + it("rotation-fails-closed: a response without a refresh token revokes, 410, and the old token is never presented again", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "no_refresh_token"; + + const first = await rig.resolve(); + expect(first.statusCode).toBe(410); + expect(first.json).toEqual({ error: "managed_credential_revoked", provider }); + expect(first.body).not.toContain("access-norefresh"); + const row = await rig.row(); + expect(row.accessTokenEncrypted).toBeUndefined(); + expect(row.refreshTokenEncrypted).toBeUndefined(); + expect(row.tokenExpiresAt).toBeUndefined(); + + const second = await rig.resolve(); + expect(second.statusCode).toBe(410); + expect(endpoint.calls).toBe(1); + expect((await rig.audit()).filter((e) => e.eventType === "managed_credential_revoked")).toEqual([ + { eventType: "managed_credential_revoked", payload: { provider, reason: "rotation_missing" } }, + ]); + }); + } else { + it("a response without a refresh token keeps the old one for a non-rotating provider", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "no_refresh_token"; + const answer = await rig.resolve(); + expect(answer.statusCode).toBe(200); + expect(answer.json.accessToken).toBe("access-norefresh"); + expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0"); + }); + } it("a row with no access token resolves as revoked (410), a missing row as not found (404)", timeout, async () => { const rig = await makeRig(provider, { accessToken: null, refreshToken: null }); diff --git a/tests/private-command-diagnostics.test.ts b/tests/private-command-diagnostics.test.ts index 41f16f1..d9d92ed 100644 --- a/tests/private-command-diagnostics.test.ts +++ b/tests/private-command-diagnostics.test.ts @@ -30,7 +30,7 @@ function config(): HeimdallConfig { bootstrapSigningPrivateKeyOnMissing: false, tokenEncryptionKeyBase64: Buffer.alloc(32, 7).toString("base64"), appSharedSecrets: { ghostlight: secret }, appRuntimeIds: { ghostlight: ["yggdrasil-ghostlight"] }, appBackendCallbacks: {}, storage: { backend: "memory", applySchemaOnStartup: true }, - providers: { discord: { clientId: "discord-client", clientSecret: "discord-secret" }, patreon: {}, github: {}, twitch: {}, youtube: {}, spotify: {} }, + providers: { discord: { clientId: "discord-client", clientSecret: "discord-secret" }, patreon: {}, github: {}, twitch: {}, youtube: {}, spotify: {}, joystick: {} }, } as HeimdallConfig; } diff --git a/tests/security.test.ts b/tests/security.test.ts index 5b07c6d..4f3f39d 100644 --- a/tests/security.test.ts +++ b/tests/security.test.ts @@ -38,6 +38,7 @@ function createTestConfig(overrides: Partial = {}): HeimdallConf twitch: {}, youtube: {}, spotify: {}, + joystick: {}, }, ...overrides, }; diff --git a/tests/verse-witness.test.ts b/tests/verse-witness.test.ts index acd9b89..07c141f 100644 --- a/tests/verse-witness.test.ts +++ b/tests/verse-witness.test.ts @@ -31,6 +31,7 @@ describe("Heimdall Verse witness advertisement", () => { "twitch", "youtube", "spotify", + "joystick", ]); expect(advertisement.appProfiles.map((profile) => profile.slug)).toEqual([ "repixelizer", From 43b7072dfbc046ce571ca5f09c75d1fea58cd580 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:15:12 +0200 Subject: [PATCH 50/85] Postgres store: return token_expires_at as the ISO string the port promises pg returns a Date for TIMESTAMPTZ; a byte-identity check on a kept credential saw a Date where the port types a string. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index f138f06..5f0a0a1 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -184,7 +184,8 @@ function mapLinkedIdentityRow(row: LinkedIdentityRow): StoredLinkedIdentity { } if (row.token_expires_at) { - identity.tokenExpiresAt = row.token_expires_at; + // pg hands back a Date for TIMESTAMPTZ whatever the row type says; the port promises an ISO string. + identity.tokenExpiresAt = new Date(row.token_expires_at).toISOString(); } return identity; From 3eaa9ef2654d77b7e3e2d007c34cc46614a5db68 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:15:42 +0200 Subject: [PATCH 51/85] Declare Joystick in the Idunn recipe and document the locked refresh Recipe optional_environment gains the Joystick client id and secret file; service-contract documents the row lock, rotation fail-closed and the 410/502 answers; the StreamPixels profile names Heimdall as sole holder of provider refresh tokens (F5). Co-Authored-By: Claude Opus 5.5 --- deployment/idunn/recipe.toml | 2 ++ docs/app-profiles/streampixels.md | 8 ++++++-- docs/service-contract.md | 26 ++++++++++++++++++++++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/deployment/idunn/recipe.toml b/deployment/idunn/recipe.toml index 8524451..f520258 100644 --- a/deployment/idunn/recipe.toml +++ b/deployment/idunn/recipe.toml @@ -157,6 +157,8 @@ optional_environment = [ "GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID", "GC_ACCESS_PROVIDER_DISCORD_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_GITHUB_CLIENT_SECRET_FILE", + "GC_ACCESS_PROVIDER_JOYSTICK_CLIENT_ID", + "GC_ACCESS_PROVIDER_JOYSTICK_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_PATREON_CLIENT_ID", "GC_ACCESS_PROVIDER_PATREON_CLIENT_SECRET_FILE", "GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE", diff --git a/docs/app-profiles/streampixels.md b/docs/app-profiles/streampixels.md index f28b40a..703dde7 100644 --- a/docs/app-profiles/streampixels.md +++ b/docs/app-profiles/streampixels.md @@ -20,7 +20,11 @@ Corroborated by `gamecult-ops/nginx/streampixels.gamecult.org.conf:14` The current StreamPixels architecture already makes useful splits: - the web app owns the remembered viewer session -- the service owns provider OAuth exchange and identity resolution +- Heimdall owns provider OAuth exchange, identity resolution and refresh-token + custody for Twitch, YouTube and Joystick; StreamPixels never calls a provider + token endpoint and holds no provider refresh token. It asks Heimdall for a + short-lived access token (`managed-credentials/resolve`) and treats `410 + managed_credential_revoked` as "the creator must reconnect" - the overlay remains auth-free - creator memberships and operator roles are separate from general viewer state @@ -38,7 +42,7 @@ In the service-first Heimdall version, the most likely reusable slice is: - provider OAuth authority - identity-linking authority -- managed Twitch/YouTube credential custody for creator connections +- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider) - entitlement refresh - shared claim semantics diff --git a/docs/service-contract.md b/docs/service-contract.md index c639178..47fa7fa 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -118,6 +118,7 @@ Supported providers in the skeleton: - `twitch` - `youtube` - `spotify` +- `joystick` Request body: @@ -430,9 +431,11 @@ Important behavior: - the refresh token is signed by Heimdall and scoped to one app session - the access token remains short-lived; app backends still verify it locally -- if a stored provider access token is expired or near expiry, Heimdall uses - the stored provider refresh token and persists any rotation before - evaluating entitlements +- if a stored provider access token is expired or near expiry, Heimdall takes + a row lock on the linked identity, re-reads it, uses the stored provider + refresh token and persists any rotation before evaluating entitlements; an + identity whose grant the provider has revoked (no stored access token) + contributes no entitlement - any app authenticated via `x-heimdall-app-secret` (`resolveAppCaller`) may send `entitlementPolicies`; Heimdall matches each stored identity to the policy for that provider. Sending policy without authenticating is @@ -484,7 +487,22 @@ Important behavior: - it returns access tokens only; refresh tokens remain in Heimdall custody - when a stored managed credential is expired or within the refresh window, Heimdall refreshes it before returning the access token and persists rotated - access/refresh tokens + access/refresh tokens. The refresh runs under a row lock on the linked + identity (`SELECT ... FOR UPDATE`), so a given refresh token is presented to + the provider once across concurrent requests and across two Heimdall + processes sharing one database. The provider call is bounded at 10 seconds +- a provider whose refresh tokens rotate (Joystick) fails closed: a refresh + response without a new refresh token revokes the credential instead of + keeping the old token +- `410 {"error":"managed_credential_revoked","provider":...}`: the provider + rejected the grant (`invalid_grant`), or the identity holds no access token. + The stored tokens are cleared and an audit event `managed_credential_revoked` + is written; the owner must reconnect. `404 managed_credential_not_found` + means no linked identity exists +- `502 {"error":"managed_credential_refresh_failed","provider":...}`: the + refresh failed for any reason that is not a grant verdict (invalid client, + provider 5xx, timeout, malformed body). Stored tokens are untouched and a + later request may succeed - app-local binding, diagnostics, subscription setup, and runtime behavior stay app-owned From 7ce9588503c2a07e92d1bc74ac9a25402812bbe4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:18:12 +0200 Subject: [PATCH 52/85] Postgres store: return created_at and updated_at as ISO strings too The credential write returns updated_at; a Date there broke the equality the port promises against the in-memory store. Co-Authored-By: Claude Opus 5.5 --- src/store/postgres.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/store/postgres.ts b/src/store/postgres.ts index 5f0a0a1..487f375 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -159,8 +159,8 @@ function mapLinkedIdentityRow(row: LinkedIdentityRow): StoredLinkedIdentity { providerUserId: row.provider_user_id, scopes: parseScopes(row.scopes), profileJson: row.profile_json, - createdAt: row.created_at, - updatedAt: row.updated_at, + createdAt: new Date(row.created_at).toISOString(), + updatedAt: new Date(row.updated_at).toISOString(), }; if (row.username) { @@ -184,7 +184,7 @@ function mapLinkedIdentityRow(row: LinkedIdentityRow): StoredLinkedIdentity { } if (row.token_expires_at) { - // pg hands back a Date for TIMESTAMPTZ whatever the row type says; the port promises an ISO string. + // pg hands back a Date for TIMESTAMPTZ whatever the row type says; the port promises ISO strings. identity.tokenExpiresAt = new Date(row.token_expires_at).toISOString(); } From f6baadec0c8cf1c0b10116740d174268e2219925 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:20:18 +0200 Subject: [PATCH 53/85] Tests: refresh failures surface as the typed ManagedCredentialRefreshFailed; seven providers Co-Authored-By: Claude Opus 5.5 --- tests/private-command-diagnostics.test.ts | 4 ++-- tests/verse-runtime.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/private-command-diagnostics.test.ts b/tests/private-command-diagnostics.test.ts index d9d92ed..d1801a5 100644 --- a/tests/private-command-diagnostics.test.ts +++ b/tests/private-command-diagnostics.test.ts @@ -98,7 +98,7 @@ it("keeps a provider's malformed 2xx refresh body out of every plaintext part of ); expect(reply.status).toBe("denied"); - expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ProviderResponseError, provider_body_not_json)."]); + expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ManagedCredentialRefreshFailed, managed_credential_refresh_failed)."]); expect(plaintext(reply)).not.toContain("CANARY"); }); @@ -106,7 +106,7 @@ it("keeps a provider's refusal body out of every plaintext part of the reply", a const reply = await refreshAgainst(() => new Response('CANARYrefused {"error":"invalid_grant"}', { status: 400 })); expect(reply.status).toBe("denied"); - expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ProviderResponseError, provider_status)."]); + expect(reply.diagnostics).toEqual(["Heimdall denied the private command (ManagedCredentialRefreshFailed, managed_credential_refresh_failed)."]); expect(plaintext(reply)).not.toContain("CANARY"); }); diff --git a/tests/verse-runtime.test.ts b/tests/verse-runtime.test.ts index 85182dc..3196cfa 100644 --- a/tests/verse-runtime.test.ts +++ b/tests/verse-runtime.test.ts @@ -71,7 +71,7 @@ describe("Heimdall verse runtime publication", () => { expect(bySchema.get("idunn.daemon_health")).toEqual([ "yggdrasil-heimdall", "active", - "Heimdall auth runtime active; storage=postgres; providers=2/6; apps=4; healthTransport=CultNet/RUDP", + "Heimdall auth runtime active; storage=postgres; providers=2/7; apps=4; healthTransport=CultNet/RUDP", "2026-06-16T17:10:00.000Z", "heimdall.cultnet-rudp-provider-health", "daemon-published", From 9b3d44f56a677bff28be23543a5cba4eb5685668 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:25:01 +0200 Subject: [PATCH 54/85] Tests: satisfy exactOptionalPropertyTypes in the Joystick stub Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app.test.ts b/tests/app.test.ts index b2960fc..9677466 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1374,7 +1374,7 @@ describe("Heimdall service", () => { }); function stubJoystick(identity: Record) { - const calls: Array<{ url: string; headers: Record; body?: URLSearchParams }> = []; + const calls: Array<{ url: string; headers: Record; body?: URLSearchParams | undefined }> = []; globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { const url = String(input); calls.push({ url, headers: (init?.headers ?? {}) as Record, body: init?.body as URLSearchParams | undefined }); From 6b8c4a9c63540775d4aa3a4cfba1f4f2f592554c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:47:37 +0200 Subject: [PATCH 55/85] Tests: pin the refresh fast paths, boundaries and connection hygiene Added after a diff-scoped StrykerJS run left survivors: no-lock paths, the skew boundary, decision kinds, a vanished row, open-transaction hygiene on the live store, and the token request's method, signal, token type and scope handling. Co-Authored-By: Claude Opus 5.5 --- tests/managed-credential.test.ts | 171 +++++++++++++++++++++++++++ tests/provider-token-request.test.ts | 48 ++++++++ 2 files changed, 219 insertions(+) diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 427b2c8..a3b6c75 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -125,6 +125,7 @@ interface Rig { custody: ReturnType; config: HeimdallConfig; provider: Provider; + databaseUrl: string | undefined; row(): Promise; audit(): Promise>; resolve(appIndex?: number): Promise<{ statusCode: number; json: Record; body: string }>; @@ -179,6 +180,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag const custody = createTokenCustody(config); const stores: HeimdallStore[] = []; let adminForDb: Pool | undefined; + let databaseUrl: string | undefined; if (!live) { stores.push(new InMemoryStore()); } else { @@ -190,6 +192,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag pool.on("error", () => undefined); stores.push(new PostgresStore(pool)); } + databaseUrl = urlFor(name); adminForDb = new Pool({ connectionString: urlFor(name), max: 1 }); adminForDb.on("error", () => undefined); cleanup.push(async () => { @@ -235,6 +238,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag custody, config, provider, + databaseUrl, async row() { const row = await stores[0]!.findStoredLinkedIdentityForAccount(account.id, provider); return row!; @@ -504,6 +508,37 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag expect(await rig.row()).toEqual(revoked); }); + it.runIf(live)("leaves no transaction open after a missing row or a throwing decide", timeout, async () => { + const rig = await makeRig("twitch"); + const pool = new Pool({ connectionString: rig.databaseUrl, max: 1 }); + pool.on("error", () => undefined); + try { + const store = new PostgresStore(pool); + const idle = async () => { + await sleep(10); + const result = await pool.query<{ fresh: boolean }>("SELECT now() = statement_timestamp() AS fresh"); + return result.rows[0]!.fresh; + }; + expect(await store.updateLinkedIdentityCredential(randomUUID(), async () => ({ kind: "keep" }))).toBeNull(); + expect(await idle()).toBe(true); + await expect( + store.updateLinkedIdentityCredential(rig.identityId, async () => { + throw new Error("boom"); + }) + ).rejects.toThrow("boom"); + expect(await idle()).toBe(true); + } finally { + await pool.end(); + } + }); + + it.runIf(!live)("drops a drained row's chain entry", async () => { + const rig = await makeRig("twitch"); + await Promise.all([1, 2, 3].map(() => rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async () => ({ kind: "keep" })))); + await sleep(5); + expect((rig.stores[0] as unknown as { credentialChains: Map }).credentialChains.size).toBe(0); + }); + it("serialises deciders on one row: each sees the previous commit", timeout, async () => { const rig = await makeRig("twitch"); const order: string[] = []; @@ -531,6 +566,142 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag }); }); +describe("refreshManagedCredential without the lock", () => { + const config = testConfig(); + const custody = createTokenCustody(config); + const now = Date.parse("2026-10-19T12:00:00.000Z"); + + function identity(overrides: Partial = {}, provider: Provider = "twitch"): StoredLinkedIdentity { + return { + id: "id-1", + accountId: "acct-1", + provider, + providerUserId: "u1", + scopes: ["s"], + profileJson: {}, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + accessTokenEncrypted: custody.encrypt("access-0"), + refreshTokenEncrypted: custody.encrypt("refresh-0"), + tokenExpiresAt: new Date(now + 60 * 60 * 1000).toISOString(), + ...overrides, + }; + } + + function countingStore(locked: StoredLinkedIdentity | null) { + const log = { locks: 0, decisions: [] as unknown[], audits: [] as unknown[] }; + const store = { + async updateLinkedIdentityCredential(_id: string, decide: (row: StoredLinkedIdentity) => Promise) { + log.locks += 1; + if (locked === null) return null; + log.decisions.push(await decide(locked)); + return locked; + }, + async createAuditEvent(event: unknown) { + log.audits.push(event); + }, + }; + return { store: store as unknown as HeimdallStore, log }; + } + + const ctxFor = (store: HeimdallStore) => ({ config, store, tokenCustody: custody, oauthRuntimes: createOAuthRuntimeRegistry(), clock: () => now }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("a snapshot with no access token is revoked without a lock or a provider call", async () => { + const { store, log } = countingStore(null); + const endpoint = installTokenEndpoint("twitch", "refresh-0"); + const outcome = await refreshManagedCredential(ctxFor(store), identity({ accessTokenEncrypted: undefined as never })); + expect(outcome.kind).toBe("revoked"); + expect(log.locks).toBe(0); + expect(endpoint.calls).toBe(0); + }); + + it.each([ + ["outside the skew", {}, "twitch"], + ["without an expiry", { tokenExpiresAt: undefined as never }, "twitch"], + ["without a refresh token", { refreshTokenEncrypted: undefined as never, tokenExpiresAt: new Date(now).toISOString() }, "twitch"], + ["for a provider whose runtime cannot refresh", { tokenExpiresAt: new Date(now).toISOString() }, "github"], + ] as const)("a credential %s is current with no lock and no provider call", async (_name, overrides, provider) => { + const { store, log } = countingStore(null); + const endpoint = installTokenEndpoint("twitch", "refresh-0"); + const snapshot = identity(overrides as Partial, provider); + const outcome = await refreshManagedCredential(ctxFor(store), snapshot); + expect(outcome).toEqual({ kind: "current", identity: snapshot }); + expect(log.locks).toBe(0); + expect(endpoint.calls).toBe(0); + }); + + it("an expiry exactly one skew away is due; one millisecond further is not", async () => { + const due = identity({ tokenExpiresAt: new Date(now + MANAGED_CREDENTIAL_REFRESH_SKEW_MS).toISOString() }); + const notDue = identity({ tokenExpiresAt: new Date(now + MANAGED_CREDENTIAL_REFRESH_SKEW_MS + 1).toISOString() }); + const a = countingStore(due); + installTokenEndpoint("twitch", "refresh-0"); + await refreshManagedCredential(ctxFor(a.store), due); + expect(a.log.locks).toBe(1); + const b = countingStore(notDue); + await refreshManagedCredential(ctxFor(b.store), notDue); + expect(b.log.locks).toBe(0); + }); + + it("decides revoke for a grant verdict and for a missing rotated token, rotate otherwise", async () => { + const due = identity({ tokenExpiresAt: new Date(now).toISOString() }); + const grant = countingStore(due); + installTokenEndpoint("twitch", "refresh-0").mode = "invalid_grant"; + await refreshManagedCredential(ctxFor(grant.store), due); + expect(grant.log.decisions).toEqual([{ kind: "revoke", updatedAt: new Date(now).toISOString() }]); + expect(grant.log.audits).toEqual([ + expect.objectContaining({ eventType: "managed_credential_revoked", eventPayloadJson: { provider: "twitch", reason: "provider_revoked" } }), + ]); + + const rotating = identity({ tokenExpiresAt: new Date(now).toISOString() }, "joystick"); + const missing = countingStore(rotating); + installTokenEndpoint("joystick", "refresh-0").mode = "no_refresh_token"; + await refreshManagedCredential(ctxFor(missing.store), rotating); + expect(missing.log.decisions).toEqual([{ kind: "revoke", updatedAt: new Date(now).toISOString() }]); + + const fine = countingStore(due); + installTokenEndpoint("twitch", "refresh-0"); + await refreshManagedCredential(ctxFor(fine.store), due); + expect(fine.log.decisions).toEqual([expect.objectContaining({ kind: "rotate", refreshTokenEncrypted: expect.any(String), scopes: ["s"] })]); + expect(fine.log.audits).toEqual([]); + }); + + it("a row that vanished is revoked with its tokens cleared", async () => { + const due = identity({ tokenExpiresAt: new Date(now).toISOString() }); + const { store } = countingStore(null); + installTokenEndpoint("twitch", "refresh-0"); + const outcome = await refreshManagedCredential(ctxFor(store), due); + expect(outcome.kind).toBe("revoked"); + expect(outcome.identity.accessTokenEncrypted).toBeUndefined(); + expect(outcome.identity.refreshTokenEncrypted).toBeUndefined(); + expect(outcome.identity.tokenExpiresAt).toBeUndefined(); + expect(outcome.identity.providerUserId).toBe("u1"); + }); + + it("a row revoked by someone else between the snapshot and the lock is revoked, with no provider call", async () => { + const due = identity({ tokenExpiresAt: new Date(now).toISOString() }); + const gone = identity({ accessTokenEncrypted: undefined as never, refreshTokenEncrypted: undefined as never, tokenExpiresAt: undefined as never }); + const { store } = countingStore(gone); + const endpoint = installTokenEndpoint("twitch", "refresh-0"); + const outcome = await refreshManagedCredential(ctxFor(store), due); + expect(outcome.kind).toBe("revoked"); + expect(endpoint.calls).toBe(0); + }); + + it("names the provider in the failure and carries nothing the provider said", async () => { + const due = identity({ tokenExpiresAt: new Date(now).toISOString() }); + const { store } = countingStore(due); + installTokenEndpoint("twitch", "refresh-0").mode = "server_error"; + const failure = await refreshManagedCredential(ctxFor(store), due).catch((error: unknown) => error as ManagedCredentialRefreshFailed); + expect(failure).toBeInstanceOf(ManagedCredentialRefreshFailed); + expect((failure as Error).message).toBe("Twitch managed credential refresh failed."); + expect((failure as ManagedCredentialRefreshFailed).code).toBe("managed_credential_refresh_failed"); + }); +}); + describe("live Postgres availability", () => { it.skipIf(!!adminUrl)("is present whenever the job requires it", () => { // Local workstation runs skip the live suites; the verification job sets diff --git a/tests/provider-token-request.test.ts b/tests/provider-token-request.test.ts index ff13d72..875598e 100644 --- a/tests/provider-token-request.test.ts +++ b/tests/provider-token-request.test.ts @@ -18,6 +18,7 @@ function configFor(): HeimdallConfig { interface Captured { url: string; + method: string | undefined; headers: Record; body: URLSearchParams; } @@ -27,6 +28,7 @@ function capture(answer: () => Response): { calls: Captured[] } { globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { calls.push({ url: String(input), + method: init?.method, headers: init?.headers as Record, body: init?.body as URLSearchParams, }); @@ -58,6 +60,7 @@ describe("requestProviderTokens follows the provider descriptor", () => { expect(calls).toHaveLength(1); const call = calls[0]!; expect(call.url).toBe(descriptor.tokenEndpoint); + expect(call.method).toBe("POST"); expect(call.headers["Content-Type"]).toBe("application/x-www-form-urlencoded"); expect(call.headers.Accept).toBe("application/json"); expect(call.body.get("grant_type")).toBe(grantType); @@ -108,6 +111,51 @@ describe("requestProviderTokens follows the provider descriptor", () => { ); }); + it("passes the abort signal to the provider call", async () => { + let seen: AbortSignal | null | undefined; + globalThis.fetch = (async (_input: string | URL, init?: RequestInit) => { + seen = init?.signal; + return ok(); + }) as typeof fetch; + const controller = new AbortController(); + await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" }, controller.signal); + expect(seen).toBe(controller.signal); + }); + + it("keeps the provider's token type and defaults a missing one to Bearer", async () => { + capture(() => new Response(JSON.stringify({ access_token: "a", token_type: "bearer" }), { status: 200 })); + expect((await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" })).tokenType).toBe("bearer"); + capture(() => new Response(JSON.stringify({ access_token: "a" }), { status: 200 })); + expect((await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" })).tokenType).toBe("Bearer"); + }); + + it("trims scope entries and drops empty ones", async () => { + capture(() => new Response(JSON.stringify({ access_token: "a", scope: [" a ", "", "b"] }), { status: 200 })); + expect((await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" })).scope).toEqual(["a", "b"]); + }); + + it("treats an empty 2xx body as an empty answer, not a malformed one", async () => { + capture(() => new Response("", { status: 200 })); + const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" }); + expect(tokens.accessToken).toBeUndefined(); + }); + + it("sets no refreshToken key when the provider sent none", async () => { + capture(() => new Response(JSON.stringify({ access_token: "a" }), { status: 200 })); + const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" }); + expect("refreshToken" in tokens).toBe(false); + }); + + it.each([ + ["only the secret missing", { clientId: "id" }], + ["only the id missing", { clientSecret: "secret" }], + ])("refuses a provider with %s", async (_n, partial) => { + const config = { providers: { twitch: partial } } as unknown as HeimdallConfig; + await expect(requestProviderTokens("twitch", config, { grant_type: "refresh_token", refresh_token: "r" })).rejects.toThrow( + "Twitch OAuth is not fully configured." + ); + }); + it("refuses an unconfigured provider by display name", async () => { const config = { providers: { patreon: {} } } as unknown as HeimdallConfig; await expect( From 34a8c6a81d23f7468152f1ed6398ababa2254764 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 11:52:27 +0200 Subject: [PATCH 56/85] Tests: a late decider still queues behind the one waiting on the in-memory chain Co-Authored-By: Claude Opus 5.5 --- tests/managed-credential.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index a3b6c75..f9a9c88 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -532,6 +532,23 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag } }); + it.runIf(!live)("a decider arriving after the first drains still waits for the one queued behind it", async () => { + const rig = await makeRig("twitch"); + const events: string[] = []; + const decider = (name: string, ms: number) => async (locked: StoredLinkedIdentity) => { + events.push(`start ${name}`); + await sleep(ms); + events.push(`end ${name}`); + return { kind: "keep" as const, seen: locked }; + }; + const a = rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, decider("A", 20)); + const b = rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, decider("B", 120)); + await a; + const c = rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, decider("C", 1)); + await Promise.all([b, c]); + expect(events).toEqual(["start A", "end A", "start B", "end B", "start C", "end C"]); + }); + it.runIf(!live)("drops a drained row's chain entry", async () => { const rig = await makeRig("twitch"); await Promise.all([1, 2, 3].map(() => rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async () => ({ kind: "keep" })))); From c554c0069728bcfb11393830a93b301649651e8c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:31:59 +0200 Subject: [PATCH 57/85] OAuth start: the profile's role lists decide which provider a mode may use sign_in and link now take only identityProviders; connect takes only managedConnectionProviders. supportsProvider(), which unioned the role lists with entitlementSources and never looked at the mode, is deleted: a Joystick sign_in for streampixels no longer starts, so a Deru sign-in cannot rewrite his managed credential row. No built-in profile had a provider reachable only through the union. Co-Authored-By: Claude Opus 5.5 --- docs/app-profiles/spotiverse.md | 2 +- docs/app-profiles/streampixels.md | 2 +- src/app-profiles.ts | 19 +++++++---- src/app.ts | 6 ++-- tests/app.test.ts | 55 +++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/docs/app-profiles/spotiverse.md b/docs/app-profiles/spotiverse.md index a10c72a..a847fba 100644 --- a/docs/app-profiles/spotiverse.md +++ b/docs/app-profiles/spotiverse.md @@ -20,7 +20,7 @@ > still reports `providers=2/6`, which counts it. > > **Why it was removed anyway:** `/v1/oauth/spotify/start` is gated by -> `supportsProvider(profile, provider)`, so a provider with no app behind it is +> `admitsProviderForMode(profile, provider, mode)`, so a provider with no app behind it is > unreachable. While the profile existed, that path was publicly reachable and > would have run a real OAuth handshake against a real client secret, taking > custody of a user's Spotify refresh token on behalf of a consumer that could diff --git a/docs/app-profiles/streampixels.md b/docs/app-profiles/streampixels.md index 703dde7..9d1e23d 100644 --- a/docs/app-profiles/streampixels.md +++ b/docs/app-profiles/streampixels.md @@ -42,7 +42,7 @@ In the service-first Heimdall version, the most likely reusable slice is: - provider OAuth authority - identity-linking authority -- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider) +- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders`, and refuses anything else before any redirect) - entitlement refresh - shared claim semantics diff --git a/src/app-profiles.ts b/src/app-profiles.ts index 88466d9..8ac57dd 100644 --- a/src/app-profiles.ts +++ b/src/app-profiles.ts @@ -3,7 +3,7 @@ import { type CapabilityDefinition, type CapabilityMode, } from "./capability-rules.js"; -import { type AppSlug, type LinkedIdentityInput, type OAuthEntitlementPolicy, type Provider } from "./contracts.js"; +import { type AppSlug, type LinkedIdentityInput, type OAuthEntitlementPolicy, type OAuthMode, type Provider } from "./contracts.js"; import { entitlementFacts, grantFacts, identityFacts } from "./facts.js"; export type { CapabilityDefinition, CapabilityMode }; @@ -222,12 +222,17 @@ export function serializeAppProfile(profile: AppProfile): Record { expect(url.searchParams.get("redirect_uri")).toBe("https://heimdall.gamecult.org/v1/oauth/joystick/callback"); }); + it.each(["sign_in", "link"] as const)( + "refuses %s for joystick on streampixels before any redirect, and Deru's managed row is untouched", + async (mode) => { + const store = new InMemoryStore(); + const app = await buildApp({ config: createTestConfig(), store }); + apps.push(app); + stubJoystick({ channel_id: "deru-chan", username: "deru" }); + const stateToken = (await connect(app)).json().stateToken as string; + const connected = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=a&state=${encodeURIComponent(stateToken)}`, + }); + const accountId = connected.json().account.id as string; + const before = await store.findStoredLinkedIdentityForAccount(accountId, "joystick"); + + const start = await app.inject({ + method: "POST", + url: "/v1/oauth/joystick/start", + payload: { appSlug: "streampixels", mode, returnTo: "https://streampixels.gamecult.org/auth/connect" }, + }); + expect(start.statusCode).toBe(400); + expect(start.json()).toMatchObject({ error: "provider_not_supported_for_app", provider: "joystick", mode }); + expect(start.json().authorizationUrl).toBeUndefined(); + expect(await store.findStoredLinkedIdentityForAccount(accountId, "joystick")).toEqual(before); + } + ); + function stubJoystick(identity: Record) { const calls: Array<{ url: string; headers: Record; body?: URLSearchParams | undefined }> = []; globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { @@ -1990,3 +2018,30 @@ describe("what may be recorded about a caught error", () => { expect(errorIdentity(error)).toEqual(expected); }); }); + +describe("OAuth start admits a provider by the profile's role for the mode", () => { + const allProviders = ["discord", "patreon", "github", "twitch", "youtube", "spotify", "joystick"] as const; + const modes = ["sign_in", "link", "connect"] as const; + + it("sign_in and link take identityProviders only; connect takes managedConnectionProviders only, for every profile", async () => { + const app = await buildApp({ config: createTestConfig() }); + try { + for (const profile of Object.values(builtInAppProfiles)) { + for (const provider of allProviders) { + for (const mode of modes) { + const start = await app.inject({ + method: "POST", + url: `/v1/oauth/${provider}/start`, + payload: { appSlug: profile.slug, mode, returnTo: `${profile.allowedReturnOrigins[0]}/x` }, + }); + const roleList = mode === "connect" ? profile.managedConnectionProviders : profile.identityProviders; + const refused = start.json().error === "provider_not_supported_for_app"; + expect([profile.slug, provider, mode, refused]).toEqual([profile.slug, provider, mode, !roleList.includes(provider)]); + } + } + } + } finally { + await app.close(); + } + }); +}); From d989860163ff1e94e68fbe5a360fee2477070aba Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:34:17 +0200 Subject: [PATCH 58/85] Refresh classification: revoke only on a 400 invalid_grant; any other answer keeps the tokens A grant verdict is read from a 400 body by exact code, never from a 401, a 5xx or a 2xx. A 2xx that is not an object with a non-empty string access_token and no error is refused as provider_body_not_token_set, so a malformed success keeps the tokens and surfaces as the typed 502 for every provider, rotating or not. A refresh answer without expires_in stores the expiry as unknown rather than keeping the already-due one; an unknown expiry is never due, so the next resolve does not refresh again. Pins Soul's three surviving mutants: substring code match, the dropped expiry rule, and the snapshot refresh token on a non-rotating provider. Co-Authored-By: Claude Opus 5.5 --- docs/service-contract.md | 10 +- src/managed-credential.ts | 5 +- src/oauth.ts | 30 +++++- tests/managed-credential.test.ts | 146 +++++++++++++++++++++++++++ tests/provider-token-request.test.ts | 37 ++++++- 5 files changed, 216 insertions(+), 12 deletions(-) diff --git a/docs/service-contract.md b/docs/service-contract.md index 47fa7fa..001b34b 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -495,14 +495,18 @@ Important behavior: response without a new refresh token revokes the credential instead of keeping the old token - `410 {"error":"managed_credential_revoked","provider":...}`: the provider - rejected the grant (`invalid_grant`), or the identity holds no access token. + rejected the grant (HTTP 400 whose OAuth `error` is exactly `invalid_grant`), + or the identity holds no access token. The stored tokens are cleared and an audit event `managed_credential_revoked` is written; the owner must reconnect. `404 managed_credential_not_found` means no linked identity exists - `502 {"error":"managed_credential_refresh_failed","provider":...}`: the refresh failed for any reason that is not a grant verdict (invalid client, - provider 5xx, timeout, malformed body). Stored tokens are untouched and a - later request may succeed + a 401 or 5xx whatever its body says, timeout, a 2xx without a string + `access_token` or carrying an `error`, an empty or non-JSON body). Stored + tokens are untouched and a later request may succeed. An answer without + `expires_in` stores the expiry as unknown, and an unknown expiry is never + due, so it does not trigger another refresh - app-local binding, diagnostics, subscription setup, and runtime behavior stay app-owned diff --git a/src/managed-credential.ts b/src/managed-credential.ts index 4da4b5a..6a06727 100644 --- a/src/managed-credential.ts +++ b/src/managed-credential.ts @@ -113,7 +113,10 @@ export async function refreshManagedCredential( refreshTokenEncrypted = locked.refreshTokenEncrypted!; } - const tokenExpiresAt = refreshed.expiresAt ?? locked.tokenExpiresAt; + // An answer without expires_in leaves the expiry unknown. The old expiry is + // already due, so keeping it would refresh again on every resolve; an + // unknown expiry is never due (dueForRefresh). + const tokenExpiresAt = refreshed.expiresAt; return { kind: "rotate", accessTokenEncrypted: ctx.tokenCustody.encrypt(refreshed.accessToken), diff --git a/src/oauth.ts b/src/oauth.ts index c46976e..9593b54 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -89,7 +89,7 @@ function readOAuthErrorCode(text: string): OAuthErrorCode | undefined { */ export class ProviderResponseError extends Error { readonly status: number; - readonly code: "provider_status" | "provider_body_not_json"; + readonly code: "provider_status" | "provider_body_not_json" | "provider_body_not_token_set"; /** * The RFC 6749 section 5.2 error code of a refusal, when the provider sent * one from that closed set. The only value taken from a provider's body. @@ -109,7 +109,14 @@ export class ProviderResponseError extends Error { async function readProviderJson(response: Response, failure: string): Promise { const text = await response.text(); if (!response.ok) { - throw new ProviderResponseError(response.status, "provider_status", failure, readOAuthErrorCode(text)); + // RFC 6749 section 5.2 sends a grant verdict as 400 only. The same JSON + // behind a 401, a 5xx or a gateway page is not one. + throw new ProviderResponseError( + response.status, + "provider_status", + failure, + response.status === 400 ? readOAuthErrorCode(text) : undefined + ); } if (!text) { return {} as T; @@ -173,13 +180,28 @@ export async function requestProviderTokens( init.signal = signal; } const label = grant.grant_type === "refresh_token" ? "refresh" : "exchange"; - const tokenResponse = await fetchJson<{ + const failure = `${descriptor.displayName} token ${label} failed`; + const response = await fetch(descriptor.tokenEndpoint, init); + const answer = await readProviderJson(response, failure); + // A 2xx is a token set only when it is an object holding a string access + // token and no error. + if ( + typeof answer !== "object" || + answer === null || + Array.isArray(answer) || + typeof (answer as { access_token?: unknown }).access_token !== "string" || + !(answer as { access_token: string }).access_token || + (answer as { error?: unknown }).error !== undefined + ) { + throw new ProviderResponseError(response.status, "provider_body_not_token_set", failure); + } + const tokenResponse = answer as { access_token: string; refresh_token?: string; token_type?: string; expires_in?: number; scope?: string | string[]; - }>(descriptor.tokenEndpoint, init, `${descriptor.displayName} token ${label} failed`); + }; const tokenSet: OAuthTokenSet = { accessToken: tokenResponse.access_token, diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index f9a9c88..7758534 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -719,6 +719,152 @@ describe("refreshManagedCredential without the lock", () => { }); }); +describe("refresh answer classification", () => { + const config = testConfig(); + const custody = createTokenCustody(config); + const now = Date.parse("2026-10-19T12:00:00.000Z"); + const dueAt = new Date(now).toISOString(); + + async function seedRow(store: InMemoryStore, provider: Provider) { + const created = new Date(now).toISOString(); + const account = await store.createAccount({ createdAt: created, lastSeenAt: created }); + return store.upsertLinkedIdentity({ + accountId: account.id, + provider, + providerUserId: `u-${randomUUID()}`, + scopes: ["s"], + profileJson: {}, + accessTokenEncrypted: custody.encrypt("access-0"), + refreshTokenEncrypted: custody.encrypt("refresh-0"), + tokenExpiresAt: dueAt, + createdAt: created, + updatedAt: created, + }); + } + + const ctxFor = (store: HeimdallStore, extra: { timeoutMs?: number } = {}) => ({ + config, + store, + tokenCustody: custody, + oauthRuntimes: createOAuthRuntimeRegistry(), + clock: () => now, + ...extra, + }); + + function answer(body: string, status: number) { + const calls: string[] = []; + globalThis.fetch = (async (_input: string | URL, init?: RequestInit) => { + calls.push(String((init!.body as URLSearchParams).get("refresh_token"))); + return new Response(body, { status }); + }) as typeof fetch; + return calls; + } + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + const grantVerdict = JSON.stringify({ error: "invalid_grant" }); + const keepsTokens: Array<[string, string, number]> = [ + ["an error code that only contains invalid_grant", JSON.stringify({ error: "invalid_grant_extra" }), 400], + ["an error code that is contained in a longer one", JSON.stringify({ error: "not_invalid_grant" }), 400], + ["a different error code", JSON.stringify({ error: "invalid_request" }), 400], + ["a 401 carrying invalid_grant", grantVerdict, 401], + ["a 500 carrying invalid_grant", grantVerdict, 500], + ["a 503 carrying invalid_grant", grantVerdict, 503], + ["a 2xx carrying invalid_grant", grantVerdict, 200], + ["a 2xx carrying an error beside an access token", JSON.stringify({ access_token: "a", error: "server_error" }), 200], + ["a 2xx with an empty object", "{}", 200], + ["a 2xx with an empty body", "", 200], + ["a 2xx with JSON null", "null", 200], + ["a 2xx with a JSON array", "[]", 200], + ["a 2xx with a non-string access token", JSON.stringify({ access_token: 7, refresh_token: "r" }), 200], + ["a 2xx with an empty access token", JSON.stringify({ access_token: "", refresh_token: "r" }), 200], + ["a 2xx that is not JSON", "", 200], + ["a 5xx page", "upstream exploded", 502], + ]; + + describe.each(["twitch", "joystick"] as const)("%s", (provider) => { + it.each(keepsTokens)("keeps the tokens and fails typed for %s", async (_name, body, status) => { + const store = new InMemoryStore(); + const row = await seedRow(store, provider); + const calls = answer(body, status); + await expect(refreshManagedCredential(ctxFor(store), row)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + expect(calls).toEqual(["refresh-0"]); + const after = (await store.findStoredLinkedIdentity(provider, row.providerUserId))!; + expect(after.accessTokenEncrypted).toBe(row.accessTokenEncrypted); + expect(after.refreshTokenEncrypted).toBe(row.refreshTokenEncrypted); + expect(after.tokenExpiresAt).toBe(dueAt); + }); + + it("keeps the tokens and fails typed when the provider never answers", async () => { + const store = new InMemoryStore(); + const row = await seedRow(store, provider); + globalThis.fetch = ((_input: string | URL, init?: RequestInit) => + new Promise((_resolve, reject) => { + init!.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); + })) as typeof fetch; + await expect(refreshManagedCredential(ctxFor(store, { timeoutMs: 30 }), row)).rejects.toBeInstanceOf(ManagedCredentialRefreshFailed); + const after = (await store.findStoredLinkedIdentity(provider, row.providerUserId))!; + expect(after.refreshTokenEncrypted).toBe(row.refreshTokenEncrypted); + }); + + it("revokes and clears the tokens on a 400 whose error is exactly invalid_grant", async () => { + const store = new InMemoryStore(); + const row = await seedRow(store, provider); + answer(grantVerdict, 400); + const outcome = await refreshManagedCredential(ctxFor(store), row); + expect(outcome.kind).toBe("revoked"); + const after = (await store.findStoredLinkedIdentity(provider, row.providerUserId))!; + expect(after.accessTokenEncrypted).toBeUndefined(); + expect(after.refreshTokenEncrypted).toBeUndefined(); + }); + }); + + it("a rotating provider's well-formed answer without a refresh token still revokes (rotation_missing)", async () => { + const store = new InMemoryStore(); + const row = await seedRow(store, "joystick"); + answer(JSON.stringify({ access_token: "a", expires_in: 3600 }), 200); + expect((await refreshManagedCredential(ctxFor(store), row)).kind).toBe("revoked"); + expect((await store.findStoredLinkedIdentity("joystick", row.providerUserId))!.refreshTokenEncrypted).toBeUndefined(); + }); + + it("an answer without expires_in stores the expiry as unknown, and the next resolve does not refresh again", async () => { + const store = new InMemoryStore(); + const row = await seedRow(store, "twitch"); + const calls = answer(JSON.stringify({ access_token: "access-1", refresh_token: "refresh-1" }), 200); + const first = await refreshManagedCredential(ctxFor(store), row); + expect(first.kind).toBe("current"); + expect(first.identity.tokenExpiresAt).toBeUndefined(); + const stored = (await store.findStoredLinkedIdentity("twitch", row.providerUserId))!; + expect(stored.tokenExpiresAt).toBeUndefined(); + expect(custody.decrypt(stored.accessTokenEncrypted!)).toBe("access-1"); + + await refreshManagedCredential(ctxFor(store), stored); + expect(calls).toEqual(["refresh-0"]); + }); + + it("a non-rotating provider keeps the locked row's refresh token, not the snapshot's", async () => { + const store = new InMemoryStore(); + const snapshot = await seedRow(store, "twitch"); + // Another caller rotated the pair after the snapshot; its expiry is still due. + await store.updateLinkedIdentityCredential(snapshot.id, async () => ({ + kind: "rotate", + accessTokenEncrypted: custody.encrypt("access-1"), + refreshTokenEncrypted: custody.encrypt("refresh-1"), + tokenExpiresAt: dueAt, + scopes: ["s"], + updatedAt: dueAt, + })); + const calls = answer(JSON.stringify({ access_token: "access-2", expires_in: 3600 }), 200); + await refreshManagedCredential(ctxFor(store), snapshot); + expect(calls).toEqual(["refresh-1"]); + const after = (await store.findStoredLinkedIdentity("twitch", snapshot.providerUserId))!; + expect(custody.decrypt(after.refreshTokenEncrypted!)).toBe("refresh-1"); + expect(custody.decrypt(after.accessTokenEncrypted!)).toBe("access-2"); + }); +}); + describe("live Postgres availability", () => { it.skipIf(!!adminUrl)("is present whenever the job requires it", () => { // Local workstation runs skip the live suites; the verification job sets diff --git a/tests/provider-token-request.test.ts b/tests/provider-token-request.test.ts index 875598e..501a6e3 100644 --- a/tests/provider-token-request.test.ts +++ b/tests/provider-token-request.test.ts @@ -134,10 +134,26 @@ describe("requestProviderTokens follows the provider descriptor", () => { expect((await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" })).scope).toEqual(["a", "b"]); }); - it("treats an empty 2xx body as an empty answer, not a malformed one", async () => { - capture(() => new Response("", { status: 200 })); - const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" }); - expect(tokens.accessToken).toBeUndefined(); + it.each([ + ["an empty body", ""], + ["an empty object", "{}"], + ["JSON null", "null"], + ["a JSON array", "[]"], + ["an error beside an access token", JSON.stringify({ access_token: "a", error: "invalid_grant" })], + ["an empty access token", JSON.stringify({ access_token: "" })], + ["a numeric access token", JSON.stringify({ access_token: 5 })], + ])("refuses a 2xx with %s as not a token set, carrying nothing of the body", async (_name, body) => { + capture(() => new Response(body, { status: 200 })); + const error = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "r" }).then( + () => { + throw new Error("expected a refusal"); + }, + (e: unknown) => e as Error & { code?: string; status?: number; oauthError?: unknown } + ); + expect(error.name).toBe("ProviderResponseError"); + expect(error.code).toBe("provider_body_not_token_set"); + expect(error.status).toBe(200); + expect(error.oauthError).toBeUndefined(); }); it("sets no refreshToken key when the provider sent none", async () => { @@ -182,6 +198,19 @@ describe("oauthError is a closed set", () => { expect(`${error.message} ${error.stack} ${inspect(error, { depth: 5, showHidden: true })} ${JSON.stringify(error)}`).not.toContain("CANARY"); }); + it.each([401, 403, 429, 500, 502, 503])("takes no oauthError from a %i, whatever its body says", async (status) => { + const error = await failure(JSON.stringify({ error: "invalid_grant" }), status); + expect(error.oauthError).toBeUndefined(); + }); + + it.each(["invalid_grant_extra", "not_invalid_grant", "INVALID_GRANT", " invalid_grant", "invalid_grant "])( + "takes the code by exact match only: %j is no code", + async (code) => { + const error = await failure(JSON.stringify({ error: code })); + expect(error.oauthError).toBeUndefined(); + } + ); + it.each([ ["an error outside the RFC set", JSON.stringify({ error: "CANARY_unknown" })], ["a non-string error", JSON.stringify({ error: { code: "invalid_grant" } })], From b15bf33b21b548dda18c803fef6b2758d6cc204f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:34:39 +0200 Subject: [PATCH 59/85] Refresh: one attempt per identity per process; waiters share its outcome Callers that arrive while a refresh of one linked identity is in flight in this process join it and receive its success or failure, instead of queuing on the row lock and presenting the same refresh token again behind a failed holder. The entry lives as long as the attempt, so a later request retries. The store lock stays the cross-process guard; the residual risk (one presentation per process per burst) is written into the service contract. Co-Authored-By: Claude Opus 5.5 --- docs/service-contract.md | 7 +++++- src/managed-credential.ts | 43 +++++++++++++++++++++++++++++--- tests/managed-credential.test.ts | 25 +++++++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/docs/service-contract.md b/docs/service-contract.md index 001b34b..2a8a47f 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -490,7 +490,12 @@ Important behavior: access/refresh tokens. The refresh runs under a row lock on the linked identity (`SELECT ... FOR UPDATE`), so a given refresh token is presented to the provider once across concurrent requests and across two Heimdall - processes sharing one database. The provider call is bounded at 10 seconds + processes sharing one database. Within one process, concurrent refreshes of + one identity share a single attempt and its outcome (success or failure), so + nobody re-presents a token behind a failed holder in that burst. Residual + risk, accepted: the guard is one presentation per process per burst, so N + Heimdall processes can present one token N times after a transient failure. + The provider call is bounded at 10 seconds - a provider whose refresh tokens rotate (Joystick) fails closed: a refresh response without a new refresh token revokes the credential instead of keeping the old token diff --git a/src/managed-credential.ts b/src/managed-credential.ts index 6a06727..617a897 100644 --- a/src/managed-credential.ts +++ b/src/managed-credential.ts @@ -60,18 +60,31 @@ function withoutTokens(identity: StoredLinkedIdentity): StoredLinkedIdentity { return rest; } +/** + * Refreshes in flight in this process, per store and linked identity. The row + * lock serialises processes; it cannot tell a waiter that queued behind a + * failed holder from a caller that arrived after the failure, so each waiter + * would present the same refresh token again. Callers that arrive while an + * attempt is in flight join it and receive its outcome, success or failure. + * The entry lives exactly as long as the attempt: a later request retries. + * + * Accepted residual risk (docs/service-contract.md): one presentation per + * process per burst, so N processes may present one token N times. + */ +const refreshesInFlight = new WeakMap>>(); + /** * The one refresh policy. `snapshot` is a hint: it may skip the lock when no * refresh is due and never decides one. The row's lock owns when a refresh - * token is presented; the decision is made on the row as read under it. + * token is presented across processes; this process makes one attempt per + * identity at a time; the decision is made on the row as read under the lock. */ export async function refreshManagedCredential( ctx: ManagedCredentialContext, snapshot: StoredLinkedIdentity ): Promise { const clock = ctx.clock ?? Date.now; - const provider = snapshot.provider; - const runtime = ctx.oauthRuntimes[provider]; + const runtime = ctx.oauthRuntimes[snapshot.provider]; if (!snapshot.accessTokenEncrypted) { return { kind: "revoked", identity: snapshot }; @@ -80,6 +93,30 @@ export async function refreshManagedCredential( return { kind: "current", identity: snapshot }; } + let inFlight = refreshesInFlight.get(ctx.store); + if (!inFlight) { + inFlight = new Map(); + refreshesInFlight.set(ctx.store, inFlight); + } + const joined = inFlight.get(snapshot.id); + if (joined) { + return joined; + } + const attempt = refreshUnderLock(ctx, snapshot).finally(() => { + inFlight.delete(snapshot.id); + }); + inFlight.set(snapshot.id, attempt); + return attempt; +} + +async function refreshUnderLock( + ctx: ManagedCredentialContext, + snapshot: StoredLinkedIdentity +): Promise { + const clock = ctx.clock ?? Date.now; + const provider = snapshot.provider; + const runtime = ctx.oauthRuntimes[provider]; + let revocation: ManagedCredentialRevocationReason | undefined; const decide = async (locked: StoredLinkedIdentity): Promise => { if (!locked.accessTokenEncrypted || !dueForRefresh(locked, runtime, clock())) { diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 7758534..36cc187 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -381,6 +381,31 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag expect(rig.custody.decrypt(outcome.identity.accessTokenEncrypted!)).toBe("access-1"); }); + it("single flight: concurrent refreshes against a never-answering endpoint make one provider call per process, and every caller gets the typed failure", timeout, async () => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = "hang"; + const snapshot = await rig.row(); + const ctxFor = (store: HeimdallStore) => ({ config: rig.config, store, tokenCustody: rig.custody, oauthRuntimes: createOAuthRuntimeRegistry(), timeoutMs: 150 }); + + const results = await Promise.allSettled( + Array.from({ length: 12 }, (_, i) => refreshManagedCredential(ctxFor(rig.stores[i % rig.stores.length]!), snapshot)) + ); + + expect(results.map((r) => r.status)).toEqual(Array(12).fill("rejected")); + for (const result of results) { + expect((result as PromiseRejectedResult).reason).toBeInstanceOf(ManagedCredentialRefreshFailed); + } + expect(endpoint.calls).toBe(rig.stores.length); + expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0"); + + // The burst is over: a later independent request tries again. + endpoint.mode = "normal"; + const later = await refreshManagedCredential(ctxFor(rig.stores[0]!), await rig.row()); + expect(later.kind).toBe("current"); + expect(endpoint.calls).toBe(rig.stores.length + 1); + }); + it("a snapshot taken before another caller's refresh does not decide: the locked row does", timeout, async () => { const rig = await makeRig(provider); const endpoint = installTokenEndpoint(provider, "refresh-0"); From c1fbc1261a02ac57b334fafe90c909c5387a4eb9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:38:13 +0200 Subject: [PATCH 60/85] OAuth start: connect keeps admitting identity providers, as Bifrost uses it The first gate refused connect for a provider outside managedConnectionProviders, which broke Bifrost's existing Discord connect. sign_in and link still take identityProviders only, so a managed-only provider (Joystick) can never sign anyone in or link; connect takes either list. Co-Authored-By: Claude Opus 5.5 --- docs/app-profiles/streampixels.md | 2 +- src/app-profiles.ts | 10 ++++++---- tests/app.test.ts | 7 +++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/app-profiles/streampixels.md b/docs/app-profiles/streampixels.md index 9d1e23d..cf7b602 100644 --- a/docs/app-profiles/streampixels.md +++ b/docs/app-profiles/streampixels.md @@ -42,7 +42,7 @@ In the service-first Heimdall version, the most likely reusable slice is: - provider OAuth authority - identity-linking authority -- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders`, and refuses anything else before any redirect) +- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders` or `identityProviders`, and refuses anything else before any redirect) - entitlement refresh - shared claim semantics diff --git a/src/app-profiles.ts b/src/app-profiles.ts index 8ac57dd..128b27a 100644 --- a/src/app-profiles.ts +++ b/src/app-profiles.ts @@ -225,13 +225,15 @@ export function serializeAppProfile(profile: AppProfile): Record { + it("sign_in and link take identityProviders only; connect takes managedConnectionProviders or identityProviders, for every profile", async () => { const app = await buildApp({ config: createTestConfig() }); try { for (const profile of Object.values(builtInAppProfiles)) { @@ -2034,7 +2034,10 @@ describe("OAuth start admits a provider by the profile's role for the mode", () url: `/v1/oauth/${provider}/start`, payload: { appSlug: profile.slug, mode, returnTo: `${profile.allowedReturnOrigins[0]}/x` }, }); - const roleList = mode === "connect" ? profile.managedConnectionProviders : profile.identityProviders; + const roleList = + mode === "connect" + ? [...profile.managedConnectionProviders, ...profile.identityProviders] + : profile.identityProviders; const refused = start.json().error === "provider_not_supported_for_app"; expect([profile.slug, provider, mode, refused]).toEqual([profile.slug, provider, mode, !roleList.includes(provider)]); } From 6bdaed2a9b13a0765aabe37ecbe925034e51f02f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 12:44:20 +0200 Subject: [PATCH 61/85] Token set check: drop the array clause the access_token check already decides A JSON array has no string access_token, so the typeof check refuses it; the separate Array.isArray clause was an equivalent mutant's worth of code. Co-Authored-By: Claude Opus 5.5 --- src/oauth.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/oauth.ts b/src/oauth.ts index 9593b54..39dd2c0 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -188,7 +188,6 @@ export async function requestProviderTokens( if ( typeof answer !== "object" || answer === null || - Array.isArray(answer) || typeof (answer as { access_token?: unknown }).access_token !== "string" || !(answer as { access_token: string }).access_token || (answer as { error?: unknown }).error !== undefined From 79ea0007d3d6be9b19ce6a8cf192518061eba2d9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 13:23:53 +0200 Subject: [PATCH 62/85] OAuth callback: mode and provider role decide what a completion mints The start gate only refused early. The callback ignored the signed state's mode, so a Joystick connect for StreamPixels redeemed into a viewer_access session. callbackMint(profile, provider, mode) now decides at the callback: identity providers keep a viewer session (Bifrost discord connect unchanged); a connect for a managed-connection-only provider stores custody and mints a claim with no facts, no capabilities and no refresh token (the refresh route would recompute viewer_access from the account); anything else is refused. StreamPixels' redeem reads only account, provider, mode, connection and claimSet.identities, so it accepts the capability-free claim unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/app-profiles/streampixels.md | 2 +- src/app-profiles.ts | 19 ++++++++ src/app.ts | 20 ++++++-- src/backend-handoff.ts | 8 +++- src/claims.ts | 28 +++++++---- tests/app.test.ts | 23 ++++++++- tests/managed-credential.test.ts | 78 +++++++++++++++++++++++++++++++ 7 files changed, 164 insertions(+), 14 deletions(-) diff --git a/docs/app-profiles/streampixels.md b/docs/app-profiles/streampixels.md index cf7b602..fe101eb 100644 --- a/docs/app-profiles/streampixels.md +++ b/docs/app-profiles/streampixels.md @@ -42,7 +42,7 @@ In the service-first Heimdall version, the most likely reusable slice is: - provider OAuth authority - identity-linking authority -- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders` or `identityProviders`, and refuses anything else before any redirect) +- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders` or `identityProviders`, and refuses anything else before any redirect. The callback decides again from the signed state's mode: a `connect` for a managed-connection-only provider stores the credential and mints a claim with no facts, no capabilities and no refresh token, so it can never be a viewer session) - entitlement refresh - shared claim semantics diff --git a/src/app-profiles.ts b/src/app-profiles.ts index 128b27a..621f3cb 100644 --- a/src/app-profiles.ts +++ b/src/app-profiles.ts @@ -250,3 +250,22 @@ export function isAllowedReturnOrigin(profile: AppProfile, returnTo: string): bo return false; } } + +/** + * What a completed OAuth callback may mint, decided at the callback from the + * signed state's mode and the provider's role in the profile (the start gate + * is only an early refusal). `viewer_session`: the provider is one of the + * profile's `identityProviders`, so the completion signs the account in with + * the profile's capabilities. `custody_only`: a `connect` for a provider that + * is only a managed connection (Joystick on StreamPixels) stores the creator + * credential and mints a claim with no capabilities and no refresh token. + * `refused`: any other combination, which the start gate never admits. + */ +export type CallbackMint = "viewer_session" | "custody_only" | "refused"; + +export function callbackMint(profile: AppProfile, provider: Provider, mode: OAuthMode): CallbackMint { + if (profile.identityProviders.includes(provider)) { + return "viewer_session"; + } + return mode === "connect" && profile.managedConnectionProviders.includes(provider) ? "custody_only" : "refused"; +} diff --git a/src/app.ts b/src/app.ts index 0a7dd60..a60e2d6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -23,7 +23,7 @@ import { type Provider, } from "./contracts.js"; import { issueAccessClaim, type IssueAccessClaimInput } from "./claims.js"; -import { builtInAppProfiles, getAppProfile, isAllowedReturnOrigin, serializeAppProfile, admitsProviderForMode } from "./app-profiles.js"; +import { builtInAppProfiles, getAppProfile, isAllowedReturnOrigin, serializeAppProfile, admitsProviderForMode, callbackMint } from "./app-profiles.js"; import { renderBrowserHandoffPage } from "./browser-handoff.js"; import { type HeimdallConfig, loadConfig } from "./config.js"; import { createTokenCustody, type TokenCustody } from "./custody.js"; @@ -1008,6 +1008,11 @@ export async function buildApp(options: BuildAppOptions = {}): Promise, + Partial> { status: "success"; mode: OAuthMode; account: AccountSummary; diff --git a/src/claims.ts b/src/claims.ts index 11c42ea..b762cbd 100644 --- a/src/claims.ts +++ b/src/claims.ts @@ -39,6 +39,12 @@ export interface IssueAccessClaimInput { displayName?: string; linkedIdentities: LinkedIdentityInput[]; facts: string[]; + /** + * The claim names the account and its identities and carries no facts and + * no capabilities (a managed-credential connect for a provider that is not + * an identity provider). The facts are not even `identity.authenticated`. + */ + withoutCapabilities?: boolean; sessionId?: string; accessRevision?: number; ttlSeconds?: number; @@ -80,13 +86,17 @@ export async function issueAccessClaim(options: { // would name an app no verifier can resolve a profile for. throw new Error(`Cannot issue an access claim for unknown app '${options.input.appSlug}'.`); } - const facts = normalizeFacts(options.input.accountId, options.input.linkedIdentities, options.input.facts); - const factSet = new Set(facts); - const sharedCapabilities = evaluateSharedCapabilities(profile, { - accountId: options.input.accountId, - facts: factSet, - identities: options.input.linkedIdentities, - }); + const withoutCapabilities = options.input.withoutCapabilities === true; + const facts = withoutCapabilities + ? [] + : normalizeFacts(options.input.accountId, options.input.linkedIdentities, options.input.facts); + const sharedCapabilities = withoutCapabilities + ? [] + : evaluateSharedCapabilities(profile, { + accountId: options.input.accountId, + facts: new Set(facts), + identities: options.input.linkedIdentities, + }); const issuedAt = nowEpochSeconds(); const ttlSeconds = clampTtlSeconds( @@ -176,6 +186,8 @@ export async function issueAccessClaim(options: { kid: options.keys.kid, }, sharedCapabilities, - hybridCapabilities: profile.capabilities.filter((capability) => capability.mode === "hybrid"), + hybridCapabilities: withoutCapabilities + ? [] + : profile.capabilities.filter((capability) => capability.mode === "hybrid"), }; } diff --git a/tests/app.test.ts b/tests/app.test.ts index 70ed754..b086853 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -2,7 +2,7 @@ import { inspect } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { type FastifyInstance } from "fastify"; import { buildApp, errorIdentity } from "../src/app.js"; -import { builtInAppProfiles } from "../src/app-profiles.js"; +import { builtInAppProfiles, callbackMint } from "../src/app-profiles.js"; import { deliverBackendHandoff, type BackendHandoffPayload } from "../src/backend-handoff.js"; import { type HeimdallConfig, loadConfig } from "../src/config.js"; import { entitlementFacts, identityFacts } from "../src/facts.js"; @@ -2048,3 +2048,24 @@ describe("OAuth start admits a provider by the profile's role for the mode", () } }); }); + +describe("callbackMint", () => { + const streampixels = builtInAppProfiles.streampixels!; + it.each([ + ["twitch", "sign_in", "viewer_session"], + ["twitch", "link", "viewer_session"], + ["youtube", "sign_in", "viewer_session"], + ["twitch", "connect", "viewer_session"], + ["joystick", "connect", "custody_only"], + ["joystick", "sign_in", "refused"], + ["joystick", "link", "refused"], + ["discord", "connect", "refused"], + ["discord", "sign_in", "refused"], + ] as const)("%s %s on streampixels is %s", (provider, mode, expected) => { + expect(callbackMint(streampixels, provider, mode)).toBe(expected); + }); + + it("keeps Bifrost's discord connect a viewer session", () => { + expect(callbackMint(builtInAppProfiles.bifrost!, "discord", "connect")).toBe("viewer_session"); + }); +}); diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 36cc187..8a0d4d2 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -474,6 +474,84 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag }); }); + describe("what the callback mints is decided by mode and provider role", () => { + const returnTo = "https://streampixels.gamecult.org/auth/connect"; + + function stubProviders(identity: Record) { + globalThis.fetch = (async (input: string | URL) => { + const url = String(input); + const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 }); + if (url.startsWith("https://api.joystick.tv/api/oauth/token") || Object.values(providerCatalog).some((d) => d.tokenEndpoint === url)) { + return json({ access_token: "tok-access", refresh_token: "tok-refresh-CANARY", token_type: "Bearer", expires_in: 3600 }); + } + if (url === "https://api.joystick.tv/api/v1/me/identity") return json({ channel_id: identity.joystick, username: "deru" }); + if (url === "https://api.twitch.tv/helix/users") return json({ data: [{ id: identity.twitch, login: "deru", display_name: "Deru" }] }); + if (url.startsWith("https://www.googleapis.com/youtube/v3/channels")) { + return json({ items: [{ id: identity.youtube, snippet: { title: "Deru" } }] }); + } + return new Response("unexpected", { status: 500 }); + }) as typeof fetch; + } + + async function flow(rig: Rig, provider: "joystick" | "twitch" | "youtube", mode: "connect" | "sign_in") { + const app = rig.apps[0]!; + const start = await app.inject({ method: "POST", url: `/v1/oauth/${provider}/start`, payload: { appSlug: "streampixels", mode, returnTo } }); + expect(start.statusCode).toBe(201); + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/${provider}/callback?code=c&state=${encodeURIComponent(start.json().stateToken as string)}`, + }); + expect(callback.statusCode).toBe(201); + const redeem = await app.inject({ + method: "POST", + url: "/v1/apps/streampixels/auth-completions/redeem", + payload: { completionCode: callback.json().completion.code as string }, + }); + expect(redeem.statusCode).toBe(201); + return { callback, redeem }; + } + + it("a joystick connect on streampixels stores custody and yields no viewer_access, no capability and no refresh token", timeout, async () => { + const rig = await makeRig("joystick"); + stubProviders({ joystick: "channel-1", twitch: "t-1", youtube: "y-1" }); + const { callback, redeem } = await flow(rig, "joystick", "connect"); + + for (const payload of [callback.json(), redeem.json()] as Array>) { + expect(payload.mode).toBe("connect"); + expect(payload.claimSet.capabilities).toEqual([]); + expect(payload.claimSet.facts).toEqual([]); + expect(payload.sharedCapabilities).toEqual([]); + expect(payload.hybridCapabilities).toEqual([]); + expect(payload.claimSet.identities).toEqual([expect.objectContaining({ provider: "joystick", providerUserId: "channel-1" })]); + expect(payload.account.id).toBe(rig.accountId); + expect(payload.refreshToken).toBeUndefined(); + expect(payload.refresh).toBeUndefined(); + } + expect(callback.body + redeem.body).not.toContain("viewer_access"); + expect(callback.body + redeem.body).not.toContain("tok-refresh-CANARY"); + const row = await rig.row(); + expect(rig.custody.decrypt(row.accessTokenEncrypted!)).toBe("tok-access"); + expect(rig.custody.decrypt(row.refreshTokenEncrypted!)).toBe("tok-refresh-CANARY"); + }); + + it.each(["twitch", "youtube"] as const)("a %s sign_in on streampixels still yields viewer_access and a refresh token", timeout, async (provider) => { + const rig = await makeRig("joystick"); + stubProviders({ joystick: "channel-1", twitch: "t-1", youtube: "y-1" }); + const { callback, redeem } = await flow(rig, provider, "sign_in"); + for (const payload of [callback.json(), redeem.json()] as Array>) { + expect(payload.claimSet.capabilities).toContain("viewer_access"); + expect(payload.refreshToken).toEqual(expect.any(String)); + } + }); + + it("a twitch connect on streampixels keeps its current behaviour: an identity provider still signs the viewer in", timeout, async () => { + const rig = await makeRig("joystick"); + stubProviders({ joystick: "channel-1", twitch: "t-1", youtube: "y-1" }); + const { redeem } = await flow(rig, "twitch", "connect"); + expect(redeem.json().claimSet.capabilities).toContain("viewer_access"); + }); + }); + describe("updateLinkedIdentityCredential", () => { it("returns null for a missing id and does not call decide", async () => { const rig = await makeRig("twitch"); From a32be41878cee2a967f7caf7b856dc38a188e9eb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 13:24:21 +0200 Subject: [PATCH 63/85] Token answers: a default lifetime replaces an unknown expiry; a malformed refresh_token is typed A token answer without a usable expires_in (absent, non-numeric, a string that is not a positive integer) now takes the descriptor's defaultAccessTokenLifetimeSeconds (3600 for every provider), so no issued token has a NULL expiry that dueForRefresh would never refresh. A 2xx whose refresh_token is present but not a string is the typed provider_body_not_token_set (502 managed_credential_refresh_failed, tokens kept); an empty string stays a missing token, so a rotating provider still fails closed. The expiry rule is now pinned on memory and both live Postgres rigs. Co-Authored-By: Claude Opus 5.5 --- docs/service-contract.md | 15 ++-- src/managed-credential.ts | 6 +- src/oauth.ts | 24 +++-- src/providers.ts | 13 +++ tests/managed-credential.test.ts | 128 +++++++++++++++++++++++---- tests/provider-token-request.test.ts | 68 ++++++++++++-- 6 files changed, 217 insertions(+), 37 deletions(-) diff --git a/docs/service-contract.md b/docs/service-contract.md index 2a8a47f..fdcc48f 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -493,7 +493,8 @@ Important behavior: processes sharing one database. Within one process, concurrent refreshes of one identity share a single attempt and its outcome (success or failure), so nobody re-presents a token behind a failed holder in that burst. Residual - risk, accepted: the guard is one presentation per process per burst, so N + risk, tracked as follow-up `joystick:follow_up:cross-process-failed-refresh-marker`: + the guard is one presentation per process per burst, so N Heimdall processes can present one token N times after a transient failure. The provider call is bounded at 10 seconds - a provider whose refresh tokens rotate (Joystick) fails closed: a refresh @@ -508,10 +509,14 @@ Important behavior: - `502 {"error":"managed_credential_refresh_failed","provider":...}`: the refresh failed for any reason that is not a grant verdict (invalid client, a 401 or 5xx whatever its body says, timeout, a 2xx without a string - `access_token` or carrying an `error`, an empty or non-JSON body). Stored - tokens are untouched and a later request may succeed. An answer without - `expires_in` stores the expiry as unknown, and an unknown expiry is never - due, so it does not trigger another refresh + `access_token` or carrying an `error`, a `refresh_token` that is present but + not a string, an empty or non-JSON body). Stored tokens are untouched and a + later request may succeed. An empty-string `refresh_token` counts as missing, + so a rotating provider revokes (`rotation_missing`). A token answer without a + usable `expires_in` (absent, non-numeric, or not a positive number or + positive-integer string) stores the expiry as now plus the provider's default + access-token lifetime (3600 seconds), so an expiry is never unknown and the + credential always comes due for refresh - app-local binding, diagnostics, subscription setup, and runtime behavior stay app-owned diff --git a/src/managed-credential.ts b/src/managed-credential.ts index 617a897..7d15e2c 100644 --- a/src/managed-credential.ts +++ b/src/managed-credential.ts @@ -150,9 +150,9 @@ async function refreshUnderLock( refreshTokenEncrypted = locked.refreshTokenEncrypted!; } - // An answer without expires_in leaves the expiry unknown. The old expiry is - // already due, so keeping it would refresh again on every resolve; an - // unknown expiry is never due (dueForRefresh). + // The token set always carries an expiry (the provider's, or the + // descriptor's default lifetime): the old one is already due, and an + // unknown one would never come due (dueForRefresh). const tokenExpiresAt = refreshed.expiresAt; return { kind: "rotate", diff --git a/src/oauth.ts b/src/oauth.ts index 39dd2c0..53f0f85 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -196,12 +196,19 @@ export async function requestProviderTokens( } const tokenResponse = answer as { access_token: string; - refresh_token?: string; + refresh_token?: unknown; token_type?: string; - expires_in?: number; + expires_in?: unknown; scope?: string | string[]; }; + // A refresh token that is present must be a string: anything else is a + // malformed grant answer, refused here as a typed failure rather than + // reaching custody as a TypeError. + if (tokenResponse.refresh_token !== undefined && typeof tokenResponse.refresh_token !== "string") { + throw new ProviderResponseError(response.status, "provider_body_not_token_set", failure); + } + const tokenSet: OAuthTokenSet = { accessToken: tokenResponse.access_token, tokenType: tokenResponse.token_type ?? "Bearer", @@ -211,12 +218,19 @@ export async function requestProviderTokens( if (tokenResponse.refresh_token) { tokenSet.refreshToken = tokenResponse.refresh_token; } - if (typeof tokenResponse.expires_in === "number" && tokenResponse.expires_in > 0) { - tokenSet.expiresAt = new Date(Date.now() + tokenResponse.expires_in * 1000).toISOString(); - } + // Nothing issued here has an unknown expiry: an answer without a usable + // expires_in takes the descriptor's default lifetime. + const lifetimeSeconds = usableExpiresIn(tokenResponse.expires_in) ?? descriptor.defaultAccessTokenLifetimeSeconds; + tokenSet.expiresAt = new Date(Date.now() + lifetimeSeconds * 1000).toISOString(); return tokenSet; } +/** A positive finite number, or a string that is a positive integer. */ +function usableExpiresIn(value: unknown): number | undefined { + const seconds = typeof value === "string" && /^\s*\d+\s*$/.test(value) ? Number(value) : value; + return typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0 ? seconds : undefined; +} + function sharedRuntimeExchange(provider: Provider): OAuthProviderRuntime["exchangeAuthorizationCode"] { return (options) => requestProviderTokens(provider, options.config, { diff --git a/src/providers.ts b/src/providers.ts index 9b139ab..77eebf4 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -11,6 +11,12 @@ export interface ProviderDescriptor { tokenEndpointAuthMethod: "client_secret_basic" | "client_secret_post"; /** True when a refresh response replaces the refresh token and the old one dies. */ rotatesRefreshTokens: boolean; + /** + * Seconds an access token lives when a token answer carries no usable + * `expires_in` (RFC 6749 5.1 makes it RECOMMENDED, not required). An unknown + * expiry is never stored: the credential would never come due for refresh. + */ + defaultAccessTokenLifetimeSeconds: number; defaultScopes: string[]; roles: ProviderRole[]; clientIdEnv: string; @@ -26,6 +32,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://discord.com/api/v10/oauth2/token", tokenEndpointAuthMethod: "client_secret_basic", rotatesRefreshTokens: false, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: ["identify", "guilds.members.read"], roles: ["identity", "entitlement"], clientIdEnv: "GC_ACCESS_PROVIDER_DISCORD_CLIENT_ID", @@ -38,6 +45,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://www.patreon.com/api/oauth2/token", tokenEndpointAuthMethod: "client_secret_post", rotatesRefreshTokens: false, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: ["identity", "identity[email]"], roles: ["identity", "entitlement"], clientIdEnv: "GC_ACCESS_PROVIDER_PATREON_CLIENT_ID", @@ -50,6 +58,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://github.com/login/oauth/access_token", tokenEndpointAuthMethod: "client_secret_post", rotatesRefreshTokens: false, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: ["read:user", "user:email"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_GITHUB_CLIENT_ID", @@ -62,6 +71,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://id.twitch.tv/oauth2/token", tokenEndpointAuthMethod: "client_secret_post", rotatesRefreshTokens: false, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: ["user:read:email"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_TWITCH_CLIENT_ID", @@ -74,6 +84,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://oauth2.googleapis.com/token", tokenEndpointAuthMethod: "client_secret_post", rotatesRefreshTokens: false, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: [ "openid", "email", @@ -96,6 +107,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://accounts.spotify.com/api/token", tokenEndpointAuthMethod: "client_secret_basic", rotatesRefreshTokens: false, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: [ "user-read-currently-playing", "user-read-playback-state", @@ -112,6 +124,7 @@ export const providerCatalog: Record = { tokenEndpoint: "https://api.joystick.tv/api/oauth/token", tokenEndpointAuthMethod: "client_secret_basic", rotatesRefreshTokens: true, + defaultAccessTokenLifetimeSeconds: 3600, defaultScopes: ["identity:read", "stream:read", "chat:read", "followers:read", "subscribers:read"], roles: ["identity", "managed_credential"], clientIdEnv: "GC_ACCESS_PROVIDER_JOYSTICK_CLIENT_ID", diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 8a0d4d2..931f7ef 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -53,7 +53,23 @@ function testConfig(): HeimdallConfig { } as unknown as HeimdallConfig; } -type Mode = "normal" | "invalid_grant" | "invalid_client" | "server_error" | "not_json" | "no_refresh_token" | "hang"; +type Mode = + | "normal" + | "invalid_grant" + | "invalid_client" + | "server_error" + | "not_json" + | "no_refresh_token" + | "empty_refresh_token" + | "bad_refresh_token" + | "no_expires_in" + | "unusable_expires_in" + | "string_expires_in" + | "hang"; + +function expiresInFor(mode: Mode): unknown { + return mode === "unusable_expires_in" ? "soon" : mode === "string_expires_in" ? "7200" : 3600; +} /** * A provider token endpoint with single-use refresh tokens. Presenting a @@ -95,7 +111,18 @@ function installTokenEndpoint(provider: Provider, firstRefreshToken: string) { return new Response(JSON.stringify({ access_token: "access-norefresh", token_type: "Bearer", expires_in: 3600 }), { status: 200, }); - case "normal": { + case "empty_refresh_token": + return new Response(JSON.stringify({ access_token: "access-empty", refresh_token: "", token_type: "Bearer", expires_in: 3600 }), { + status: 200, + }); + case "bad_refresh_token": + return new Response(JSON.stringify({ access_token: "access-bad", refresh_token: 42, token_type: "Bearer", expires_in: 3600 }), { + status: 200, + }); + case "normal": + case "no_expires_in": + case "unusable_expires_in": + case "string_expires_in": { if (state.installationRevoked || presented !== state.live) { state.installationRevoked = true; return new Response(JSON.stringify({ error: "invalid_grant", error_description: "reused CANARY" }), { status: 400 }); @@ -107,7 +134,7 @@ function installTokenEndpoint(provider: Provider, firstRefreshToken: string) { access_token: `access-${state.issued}`, refresh_token: state.live, token_type: "Bearer", - expires_in: 3600, + ...(state.mode === "no_expires_in" ? {} : { expires_in: expiresInFor(state.mode) }), }), { status: 200 } ); @@ -329,7 +356,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag expect(first.body + second.body + JSON.stringify(events)).not.toContain("CANARY"); }); - it.each(["invalid_client", "server_error", "not_json", "hang"] as const)( + it.each(["invalid_client", "server_error", "not_json", "bad_refresh_token", "hang"] as const)( "transient failure (%s) keeps custody: 502, tokens byte-identical, nothing leaked", timeout, async (mode) => { @@ -422,15 +449,16 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag }); if (providerCatalog[provider].rotatesRefreshTokens) { - it("rotation-fails-closed: a response without a refresh token revokes, 410, and the old token is never presented again", timeout, async () => { + it.each(["no_refresh_token", "empty_refresh_token"] as const)("rotation-fails-closed: a response with %s revokes, 410, and the old token is never presented again", timeout, async (mode) => { const rig = await makeRig(provider); const endpoint = installTokenEndpoint(provider, "refresh-0"); - endpoint.mode = "no_refresh_token"; + endpoint.mode = mode; const first = await rig.resolve(); expect(first.statusCode).toBe(410); expect(first.json).toEqual({ error: "managed_credential_revoked", provider }); expect(first.body).not.toContain("access-norefresh"); + expect(first.body).not.toContain("access-empty"); const row = await rig.row(); expect(row.accessTokenEncrypted).toBeUndefined(); expect(row.refreshTokenEncrypted).toBeUndefined(); @@ -444,17 +472,56 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag ]); }); } else { - it("a response without a refresh token keeps the old one for a non-rotating provider", timeout, async () => { + it.each(["no_refresh_token", "empty_refresh_token"] as const)("a response with %s keeps the old one for a non-rotating provider", timeout, async (mode) => { const rig = await makeRig(provider); const endpoint = installTokenEndpoint(provider, "refresh-0"); - endpoint.mode = "no_refresh_token"; + endpoint.mode = mode; const answer = await rig.resolve(); expect(answer.statusCode).toBe(200); - expect(answer.json.accessToken).toBe("access-norefresh"); + expect(answer.json.accessToken).toBe(mode === "no_refresh_token" ? "access-norefresh" : "access-empty"); expect(rig.custody.decrypt((await rig.row()).refreshTokenEncrypted!)).toBe("refresh-0"); }); } + it.each([ + ["no expires_in", "no_expires_in", 3600], + ["an unusable expires_in", "unusable_expires_in", 3600], + ["a numeric-string expires_in", "string_expires_in", 7200], + ] as const)( + "refresh-once-per-token: an answer with %s stores now plus %s s, not the old due expiry, and resolve refreshes again only once it passes", + timeout, + async (_name, mode, seconds) => { + const rig = await makeRig(provider); + const endpoint = installTokenEndpoint(provider, "refresh-0"); + endpoint.mode = mode; + + const before = Date.now(); + expect((await rig.resolve()).statusCode).toBe(200); + expect(endpoint.calls).toBe(1); + const stored = Date.parse((await rig.row()).tokenExpiresAt!); + expect(stored - before).toBeGreaterThanOrEqual(seconds * 1000 - 1000); + expect(stored - before).toBeLessThan(seconds * 1000 + 10_000); + + expect((await rig.resolve()).statusCode).toBe(200); + expect(endpoint.calls).toBe(1); + + // The stored expiry passes: the next resolve refreshes. + const past = new Date(Date.now() - 1000).toISOString(); + await rig.stores[0]!.updateLinkedIdentityCredential(rig.identityId, async (locked) => ({ + kind: "rotate", + accessTokenEncrypted: locked.accessTokenEncrypted!, + refreshTokenEncrypted: locked.refreshTokenEncrypted!, + tokenExpiresAt: past, + scopes: locked.scopes, + updatedAt: past, + })); + expect((await rig.resolve()).statusCode).toBe(200); + expect(endpoint.calls).toBe(2); + const again = Date.parse((await rig.row()).tokenExpiresAt!); + expect(again).toBeGreaterThan(Date.now() + MANAGED_CREDENTIAL_REFRESH_SKEW_MS); + } + ); + it("a row with no access token resolves as revoked (410), a missing row as not found (404)", timeout, async () => { const rig = await makeRig(provider, { accessToken: null, refreshToken: null }); const endpoint = installTokenEndpoint(provider, "refresh-0"); @@ -882,6 +949,8 @@ describe("refresh answer classification", () => { ["a 2xx with JSON null", "null", 200], ["a 2xx with a JSON array", "[]", 200], ["a 2xx with a non-string access token", JSON.stringify({ access_token: 7, refresh_token: "r" }), 200], + ["a 2xx with a numeric refresh token", JSON.stringify({ access_token: "a", refresh_token: 42 }), 200], + ["a 2xx with an object refresh token", JSON.stringify({ access_token: "a", refresh_token: { r: 1 } }), 200], ["a 2xx with an empty access token", JSON.stringify({ access_token: "", refresh_token: "r" }), 200], ["a 2xx that is not JSON", "", 200], ["a 5xx page", "upstream exploded", 502], @@ -924,27 +993,50 @@ describe("refresh answer classification", () => { }); }); - it("a rotating provider's well-formed answer without a refresh token still revokes (rotation_missing)", async () => { + it.each([ + ["without a refresh token", { access_token: "a", expires_in: 3600 }], + ["with an empty refresh token", { access_token: "a", refresh_token: "", expires_in: 3600 }], + ])("a rotating provider's well-formed answer %s still revokes (rotation_missing)", async (_n, body) => { const store = new InMemoryStore(); const row = await seedRow(store, "joystick"); - answer(JSON.stringify({ access_token: "a", expires_in: 3600 }), 200); + answer(JSON.stringify(body), 200); expect((await refreshManagedCredential(ctxFor(store), row)).kind).toBe("revoked"); expect((await store.findStoredLinkedIdentity("joystick", row.providerUserId))!.refreshTokenEncrypted).toBeUndefined(); }); - it("an answer without expires_in stores the expiry as unknown, and the next resolve does not refresh again", async () => { + it.each([ + ["no expires_in", {}], + ["a non-numeric string expires_in", { expires_in: "soon" }], + ])("an answer with %s stores now plus the default lifetime; resolve refreshes once it passes", async (_n, extra) => { const store = new InMemoryStore(); - const row = await seedRow(store, "twitch"); - const calls = answer(JSON.stringify({ access_token: "access-1", refresh_token: "refresh-1" }), 200); - const first = await refreshManagedCredential(ctxFor(store), row); + const seeded = await seedRow(store, "twitch"); + // Due against the real clock, which the token answer's expiry is computed from. + const row = (await store.updateLinkedIdentityCredential(seeded.id, async (locked) => ({ + kind: "rotate", + accessTokenEncrypted: locked.accessTokenEncrypted!, + refreshTokenEncrypted: locked.refreshTokenEncrypted!, + tokenExpiresAt: new Date().toISOString(), + scopes: locked.scopes, + updatedAt: new Date().toISOString(), + })))!; + const calls = answer(JSON.stringify({ access_token: "access-1", refresh_token: "refresh-1", ...extra }), 200); + let skew = 0; + const ctx = { ...ctxFor(store), clock: () => Date.now() + skew }; + const before = Date.now(); + const first = await refreshManagedCredential(ctx, row); expect(first.kind).toBe("current"); - expect(first.identity.tokenExpiresAt).toBeUndefined(); const stored = (await store.findStoredLinkedIdentity("twitch", row.providerUserId))!; - expect(stored.tokenExpiresAt).toBeUndefined(); + const expiry = Date.parse(stored.tokenExpiresAt!); + expect(expiry - before).toBeGreaterThanOrEqual(providerCatalog.twitch.defaultAccessTokenLifetimeSeconds * 1000); + expect(expiry - before).toBeLessThan(providerCatalog.twitch.defaultAccessTokenLifetimeSeconds * 1000 + 5_000); expect(custody.decrypt(stored.accessTokenEncrypted!)).toBe("access-1"); - await refreshManagedCredential(ctxFor(store), stored); + await refreshManagedCredential(ctx, stored); expect(calls).toEqual(["refresh-0"]); + + skew = providerCatalog.twitch.defaultAccessTokenLifetimeSeconds * 1000; + await refreshManagedCredential(ctx, stored); + expect(calls).toEqual(["refresh-0", "refresh-1"]); }); it("a non-rotating provider keeps the locked row's refresh token, not the snapshot's", async () => { diff --git a/tests/provider-token-request.test.ts b/tests/provider-token-request.test.ts index 501a6e3..ff415fd 100644 --- a/tests/provider-token-request.test.ts +++ b/tests/provider-token-request.test.ts @@ -90,14 +90,70 @@ describe("requestProviderTokens follows the provider descriptor", () => { expect(calls[1]!.body.has("code")).toBe(false); }); + const everyProvider = Object.keys(providerCatalog) as Provider[]; + + it("gives every provider a default access-token lifetime of 3600 seconds", () => { + for (const provider of everyProvider) { + expect(providerCatalog[provider].defaultAccessTokenLifetimeSeconds).toBe(3600); + } + }); + + // expires_in is RECOMMENDED (RFC 6749 5.1): unusable means the default, never an unknown expiry. it.each([ - ["no expires_in", { access_token: "a" }], - ["a zero expires_in", { access_token: "a", expires_in: 0 }], - ["a string expires_in", { access_token: "a", expires_in: "3600" }], - ])("sets no expiry and no refresh token for %s", async (_n, body) => { - capture(() => new Response(JSON.stringify(body), { status: 200 })); + ["no expires_in", undefined, 3600], + ["a zero expires_in", 0, 3600], + ["a negative expires_in", -5, 3600], + ["a null expires_in", null, 3600], + ["an object expires_in", {}, 3600], + ["a non-numeric string", "abc", 3600], + ["an empty string", "", 3600], + ["a negative string", "-1", 3600], + ["a zero string", "0", 3600], + ["a fractional string", "12.5", 3600], + ["an exponent string", "1e3", 3600], + ["a numeric string", "7200", 7200], + ["a padded numeric string", " 7200 ", 7200], + ["a number", 7200, 7200], + ])("stores now plus the right lifetime for %s, on every provider and grant", async (_n, expiresIn, seconds) => { + for (const provider of everyProvider) { + for (const grant of [ + { grant_type: "authorization_code", code: "c", redirect_uri: "u" } as const, + { grant_type: "refresh_token", refresh_token: "R" } as const, + ]) { + capture(() => new Response(JSON.stringify({ access_token: "a", ...(expiresIn === undefined ? {} : { expires_in: expiresIn }) }), { status: 200 })); + const before = Date.now(); + const tokens = await requestProviderTokens(provider, configFor(), grant); + const stored = Date.parse(tokens.expiresAt ?? ""); + expect(stored - before).toBeGreaterThanOrEqual(seconds * 1000); + expect(stored - before).toBeLessThan(seconds * 1000 + 5_000); + expect(tokens.refreshToken).toBeUndefined(); + } + } + }); + + it("takes the default lifetime for an expires_in too large to be a date", async () => { + capture(() => new Response('{"access_token":"a","expires_in":1e999}', { status: 200 })); + const before = Date.now(); + const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "R" }); + expect(Date.parse(tokens.expiresAt ?? "") - before).toBeLessThan(3600 * 1000 + 5_000); + }); + + it.each([ + ["a number", 42], + ["an object", { a: 1 }], + ["an array", ["r"]], + ["true", true], + ["null", null], + ])("refuses a 2xx whose refresh_token is %s as a malformed token set", async (_n, refreshToken) => { + capture(() => new Response(JSON.stringify({ access_token: "a", refresh_token: refreshToken, expires_in: 3600 }), { status: 200 })); + await expect( + requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "R" }) + ).rejects.toMatchObject({ code: "provider_body_not_token_set" }); + }); + + it("reads an empty-string refresh_token as no refresh token", async () => { + capture(() => new Response(JSON.stringify({ access_token: "a", refresh_token: "", expires_in: 3600 }), { status: 200 })); const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "R" }); - expect(tokens.expiresAt).toBeUndefined(); expect(tokens.refreshToken).toBeUndefined(); }); From 32208cc8e2b2e6cf946448b1af6d46212494eed6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 13:27:28 +0200 Subject: [PATCH 64/85] Pin the callback's refusal of a signed state the provider has no role for Re-signs a joystick connect state as sign_in and link with the app's own key and shows the callback refuses before the code exchange and writes nothing. Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/app.test.ts b/tests/app.test.ts index b086853..3afe0a6 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { inspect } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { type FastifyInstance } from "fastify"; @@ -7,7 +10,7 @@ import { deliverBackendHandoff, type BackendHandoffPayload } from "../src/backen import { type HeimdallConfig, loadConfig } from "../src/config.js"; import { entitlementFacts, identityFacts } from "../src/facts.js"; import { createOAuthRuntimeRegistry, ProviderResponseError, type OAuthProviderRuntime } from "../src/oauth.js"; -import { verifyJwt } from "../src/signing.js"; +import { createRuntimeKeyMaterial, signJwt, verifyJwt } from "../src/signing.js"; import { InMemoryStore } from "../src/store/index.js"; function createTestConfig(): HeimdallConfig { @@ -1420,6 +1423,37 @@ describe("Heimdall service", () => { return calls; } + it.each(["sign_in", "link"] as const)( + "the callback refuses a signed %s state for joystick on streampixels before the code exchange, and nothing is written", + async (mode) => { + // The start gate already refuses these words, so the state is re-signed with the app's own key. + const dir = mkdtempSync(join(tmpdir(), "heimdall-forge-")); + try { + const config = { ...createTestConfig(), signingPrivateKeyPath: join(dir, "signing.pem"), bootstrapSigningPrivateKeyOnMissing: true }; + const store = new InMemoryStore(); + const app = await buildApp({ config, store }); + apps.push(app); + const keys = createRuntimeKeyMaterial({ ...config, bootstrapSigningPrivateKeyOnMissing: false }); + const calls = stubJoystick({ channel_id: "chan-forged", username: "deru" }); + const stateToken = (await connect(app)).json().stateToken as string; + const verified = verifyJwt(stateToken, keys.publicKey); + expect(verified.valid).toBe(true); + const forged = signJwt({ ...(verified as { payload: Record }).payload, mode }, keys); + + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=c&state=${encodeURIComponent(forged)}`, + }); + expect(callback.statusCode).toBeGreaterThanOrEqual(400); + expect(callback.body).not.toContain("viewer_access"); + expect(calls).toEqual([]); + expect(await store.findStoredLinkedIdentity("joystick", "chan-forged")).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + } + ); + it("exchanges the code with Basic client auth and links the identity by channel id", async () => { const store = new InMemoryStore(); const app = await buildApp({ config: createTestConfig(), store }); From 655fcc7d228b29c7ab55561c9bb1abcc606735a4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 13:29:10 +0200 Subject: [PATCH 65/85] Credential rotate decisions carry a required expiry Refresh and exchange answers now type as ExpiringTokenSet, and a rotate decision requires tokenExpiresAt, so neither store has a no-expiry rule left to mutate: a NULL expiry can only be written by a revoke. Co-Authored-By: Claude Opus 5.5 --- src/managed-credential.ts | 5 ++--- src/oauth.ts | 17 ++++++++++------- src/store/in-memory.ts | 4 +--- src/store/types.ts | 2 +- tests/managed-credential.test.ts | 1 + 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/managed-credential.ts b/src/managed-credential.ts index 7d15e2c..e2fd124 100644 --- a/src/managed-credential.ts +++ b/src/managed-credential.ts @@ -150,15 +150,14 @@ async function refreshUnderLock( refreshTokenEncrypted = locked.refreshTokenEncrypted!; } - // The token set always carries an expiry (the provider's, or the + // The refreshed set always carries an expiry (the provider's, or the // descriptor's default lifetime): the old one is already due, and an // unknown one would never come due (dueForRefresh). - const tokenExpiresAt = refreshed.expiresAt; return { kind: "rotate", accessTokenEncrypted: ctx.tokenCustody.encrypt(refreshed.accessToken), refreshTokenEncrypted, - ...(tokenExpiresAt !== undefined ? { tokenExpiresAt } : {}), + tokenExpiresAt: refreshed.expiresAt, scopes: refreshed.scope.length ? refreshed.scope : locked.scopes, updatedAt, }; diff --git a/src/oauth.ts b/src/oauth.ts index 53f0f85..2ad2c76 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -41,6 +41,9 @@ export interface OAuthCallbackContext { entitlementPolicy: OAuthEntitlementPolicy | null; } +/** A token set whose access-token expiry is known: the provider's own, or the descriptor's default. */ +export type ExpiringTokenSet = OAuthTokenSet & { expiresAt: string }; + export interface OAuthProviderRuntime { exchangeAuthorizationCode(options: { config: HeimdallConfig; @@ -51,7 +54,7 @@ export interface OAuthProviderRuntime { config: HeimdallConfig; refreshToken: string; signal?: AbortSignal; - }): Promise; + }): Promise; resolveIdentity(options: { config: HeimdallConfig; accessToken: string }): Promise; evaluateEntitlements(options: { config: HeimdallConfig; @@ -156,7 +159,7 @@ export async function requestProviderTokens( config: HeimdallConfig, grant: TokenGrant, signal?: AbortSignal -): Promise { +): Promise { const descriptor = providerCatalog[provider]; const providerConfig = config.providers[provider]; if (!providerConfig.clientId || !providerConfig.clientSecret) { @@ -209,19 +212,19 @@ export async function requestProviderTokens( throw new ProviderResponseError(response.status, "provider_body_not_token_set", failure); } - const tokenSet: OAuthTokenSet = { + // Nothing issued here has an unknown expiry: an answer without a usable + // expires_in takes the descriptor's default lifetime. + const lifetimeSeconds = usableExpiresIn(tokenResponse.expires_in) ?? descriptor.defaultAccessTokenLifetimeSeconds; + const tokenSet: ExpiringTokenSet = { accessToken: tokenResponse.access_token, tokenType: tokenResponse.token_type ?? "Bearer", scope: normalizeScopes(tokenResponse.scope), + expiresAt: new Date(Date.now() + lifetimeSeconds * 1000).toISOString(), raw: tokenResponse as unknown as Record, }; if (tokenResponse.refresh_token) { tokenSet.refreshToken = tokenResponse.refresh_token; } - // Nothing issued here has an unknown expiry: an answer without a usable - // expires_in takes the descriptor's default lifetime. - const lifetimeSeconds = usableExpiresIn(tokenResponse.expires_in) ?? descriptor.defaultAccessTokenLifetimeSeconds; - tokenSet.expiresAt = new Date(Date.now() + lifetimeSeconds * 1000).toISOString(); return tokenSet; } diff --git a/src/store/in-memory.ts b/src/store/in-memory.ts index 0dea560..6a53dad 100644 --- a/src/store/in-memory.ts +++ b/src/store/in-memory.ts @@ -198,9 +198,7 @@ export class InMemoryStore implements HeimdallStore { if (decision.kind === "rotate") { next.accessTokenEncrypted = decision.accessTokenEncrypted; next.refreshTokenEncrypted = decision.refreshTokenEncrypted; - if (decision.tokenExpiresAt !== undefined) { - next.tokenExpiresAt = decision.tokenExpiresAt; - } + next.tokenExpiresAt = decision.tokenExpiresAt; next.scopes = [...decision.scopes]; } this.linkedIdentities.set(id, clone(next)); diff --git a/src/store/types.ts b/src/store/types.ts index 0145626..5307e1f 100644 --- a/src/store/types.ts +++ b/src/store/types.ts @@ -67,7 +67,7 @@ export type CredentialDecision = kind: "rotate"; accessTokenEncrypted: string; refreshTokenEncrypted: string; - tokenExpiresAt?: string; + tokenExpiresAt: string; scopes: string[]; updatedAt: string; } diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 931f7ef..7f473eb 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -738,6 +738,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag kind: "rotate", accessTokenEncrypted: `A${n}`, refreshTokenEncrypted: `R${n}`, + tokenExpiresAt: new Date(Date.now() + 3600_000).toISOString(), scopes: locked.scopes, updatedAt: new Date().toISOString(), }; From a911da93d8630823f24e69134f7d5c38b1baf3fe Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 13:34:41 +0200 Subject: [PATCH 66/85] Claims: facts are the one owner of a capability-free claim Every shared capability needs a non-empty anyOf, so an empty fact set already yields no capabilities; the explicit empty list beside it was a second owner no test could tell apart from the first. Co-Authored-By: Claude Opus 5.5 --- src/claims.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/claims.ts b/src/claims.ts index b762cbd..4440ed3 100644 --- a/src/claims.ts +++ b/src/claims.ts @@ -90,13 +90,13 @@ export async function issueAccessClaim(options: { const facts = withoutCapabilities ? [] : normalizeFacts(options.input.accountId, options.input.linkedIdentities, options.input.facts); - const sharedCapabilities = withoutCapabilities - ? [] - : evaluateSharedCapabilities(profile, { - accountId: options.input.accountId, - facts: new Set(facts), - identities: options.input.linkedIdentities, - }); + // Every shared capability needs a non-empty anyOf (capability-rules), so an + // empty fact set yields no capabilities: facts are the one owner of that. + const sharedCapabilities = evaluateSharedCapabilities(profile, { + accountId: options.input.accountId, + facts: new Set(facts), + identities: options.input.linkedIdentities, + }); const issuedAt = nowEpochSeconds(); const ttlSeconds = clampTtlSeconds( From 26f264877a76fcb3f542c20d9d639ff39a7c7847 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:04:12 +0200 Subject: [PATCH 67/85] Pin the hybrid capability listing a viewer sign-in carries Co-Authored-By: Claude Opus 5.5 --- tests/managed-credential.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 7f473eb..da7f991 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -608,6 +608,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag for (const payload of [callback.json(), redeem.json()] as Array>) { expect(payload.claimSet.capabilities).toContain("viewer_access"); expect(payload.refreshToken).toEqual(expect.any(String)); + expect(payload.hybridCapabilities.map((c: { key: string }) => c.key)).toEqual(["creator_access", "creator_admin"]); } }); From e1ac84fb70708c3a3176709bf3621b9ee6757f13 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:34:24 +0200 Subject: [PATCH 68/85] Joystick custody needs a creator binding, at start and at the callback callbackMint now takes the connection: custody_only is a connect for a managed-connection-only provider bound to connection.kind creator with a targetId. Anything else is refused at start with creator_connection_required before any redirect, and at the callback from the signed state before the code exchange. The streampixels profile note says what Heimdall guarantees and records the S1 rule that SP's redeem treats a non-creator Joystick completion as an error, never a sign-in. Co-Authored-By: Claude Opus 5.5 --- docs/app-profiles/streampixels.md | 2 +- src/app-profiles.ts | 24 +++++++++--- src/app.ts | 12 +++++- tests/app.test.ts | 61 +++++++++++++++++++++++++++++-- tests/managed-credential.test.ts | 2 +- 5 files changed, 88 insertions(+), 13 deletions(-) diff --git a/docs/app-profiles/streampixels.md b/docs/app-profiles/streampixels.md index fe101eb..dbd91f1 100644 --- a/docs/app-profiles/streampixels.md +++ b/docs/app-profiles/streampixels.md @@ -42,7 +42,7 @@ In the service-first Heimdall version, the most likely reusable slice is: - provider OAuth authority - identity-linking authority -- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders` or `identityProviders`, and refuses anything else before any redirect. The callback decides again from the signed state's mode: a `connect` for a managed-connection-only provider stores the credential and mints a claim with no facts, no capabilities and no refresh token, so it can never be a viewer session) +- managed Twitch/YouTube/Joystick credential custody for creator connections (Joystick is a managed connection only, not a viewer sign-in provider: `startOAuthFlow` admits `sign_in` and `link` only for `identityProviders` and `connect` only for `managedConnectionProviders` or `identityProviders`, and refuses anything else before any redirect. The callback decides again from the signed state's mode: a `connect` for a managed-connection-only provider is admitted only with `connection.kind` `creator` and a `targetId`, refused otherwise at start and again at the callback before the code exchange; an admitted one stores the credential and mints a claim with no facts, no capabilities and no refresh token. Heimdall's claim is therefore never a viewer session. Heimdall cannot stop an app from minting its own viewer from a completion, so StreamPixels' redeem must treat a non-creator Joystick completion as an error, never as a sign-in. This is an S1 rule, recorded for SP) - entitlement refresh - shared claim semantics diff --git a/src/app-profiles.ts b/src/app-profiles.ts index 621f3cb..3489a48 100644 --- a/src/app-profiles.ts +++ b/src/app-profiles.ts @@ -3,7 +3,7 @@ import { type CapabilityDefinition, type CapabilityMode, } from "./capability-rules.js"; -import { type AppSlug, type LinkedIdentityInput, type OAuthEntitlementPolicy, type OAuthMode, type Provider } from "./contracts.js"; +import { type AppSlug, type LinkedIdentityInput, type OAuthConnectionBinding, type OAuthEntitlementPolicy, type OAuthMode, type Provider } from "./contracts.js"; import { entitlementFacts, grantFacts, identityFacts } from "./facts.js"; export type { CapabilityDefinition, CapabilityMode }; @@ -257,15 +257,27 @@ export function isAllowedReturnOrigin(profile: AppProfile, returnTo: string): bo * is only an early refusal). `viewer_session`: the provider is one of the * profile's `identityProviders`, so the completion signs the account in with * the profile's capabilities. `custody_only`: a `connect` for a provider that - * is only a managed connection (Joystick on StreamPixels) stores the creator - * credential and mints a claim with no capabilities and no refresh token. - * `refused`: any other combination, which the start gate never admits. + * is only a managed connection (Joystick on StreamPixels), bound to a creator + * target, stores the creator credential and mints a claim with no capabilities + * and no refresh token. Without a creator binding it is `refused`, as is any + * other combination. The start gate asks this same question, so an unbound + * connect never reaches a redirect. */ export type CallbackMint = "viewer_session" | "custody_only" | "refused"; -export function callbackMint(profile: AppProfile, provider: Provider, mode: OAuthMode): CallbackMint { +export function callbackMint( + profile: AppProfile, + provider: Provider, + mode: OAuthMode, + connection: OAuthConnectionBinding | null | undefined +): CallbackMint { if (profile.identityProviders.includes(provider)) { return "viewer_session"; } - return mode === "connect" && profile.managedConnectionProviders.includes(provider) ? "custody_only" : "refused"; + return mode === "connect" && + profile.managedConnectionProviders.includes(provider) && + connection?.kind === "creator" && + connection.targetId !== "" + ? "custody_only" + : "refused"; } diff --git a/src/app.ts b/src/app.ts index a60e2d6..40ceb20 100644 --- a/src/app.ts +++ b/src/app.ts @@ -501,6 +501,16 @@ export async function startOAuthFlow( }; } + if (callbackMint(profile, provider, input.mode, input.connection) === "refused") { + return { + statusCode: 400, + body: { + error: "creator_connection_required", + detail: `${provider} on ${profile.slug} is a managed creator connection: connect needs connection.kind creator and a targetId.`, + }, + }; + } + if (!isAllowedReturnOrigin(profile, input.returnTo)) { return { statusCode: 400, @@ -1009,7 +1019,7 @@ export async function buildApp(options: BuildAppOptions = {}): Promise { }); describe("joystick connect", () => { - async function connect(app: FastifyInstance) { + const creator = { kind: "creator", targetId: "creator-1" }; + async function connect(app: FastifyInstance, extra: Record = { connection: creator }) { const start = await app.inject({ method: "POST", url: "/v1/oauth/joystick/start", - payload: { appSlug: "streampixels", mode: "connect", returnTo: "https://streampixels.gamecult.org/auth/connect" }, + payload: { appSlug: "streampixels", mode: "connect", returnTo: "https://streampixels.gamecult.org/auth/connect", ...extra }, }); return start; } + it.each([ + ["no connection", {}], + ["an account connection", { connection: { kind: "account", targetId: "victim-1" } }], + ["an empty targetId", { connection: { kind: "creator", targetId: "" } }], + ])("refuses a joystick connect with %s at start, before any redirect", async (_name, extra) => { + const app = await buildApp({ config: createTestConfig() }); + apps.push(app); + const start = await connect(app, extra); + expect(start.statusCode).toBe(400); + expect(start.json().authorizationUrl).toBeUndefined(); + expect(start.json().stateToken).toBeUndefined(); + }); + + it.each([ + ["no connection", null], + ["an account connection", { kind: "account", targetId: "victim-1" }], + ["an empty targetId", { kind: "creator", targetId: "" }], + ])("the callback refuses a signed joystick connect with %s before the code exchange, and nothing is written", async (_name, connection) => { + const dir = mkdtempSync(join(tmpdir(), "heimdall-forge-")); + try { + const config = { ...createTestConfig(), signingPrivateKeyPath: join(dir, "signing.pem"), bootstrapSigningPrivateKeyOnMissing: true }; + const store = new InMemoryStore(); + const app = await buildApp({ config, store }); + apps.push(app); + const keys = createRuntimeKeyMaterial({ ...config, bootstrapSigningPrivateKeyOnMissing: false }); + const calls = stubJoystick({ channel_id: "chan-forged", username: "deru" }); + const stateToken = (await connect(app)).json().stateToken as string; + const verified = verifyJwt(stateToken, keys.publicKey); + expect(verified.valid).toBe(true); + const forged = signJwt({ ...(verified as { payload: Record }).payload, connection }, keys); + + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=c&state=${encodeURIComponent(forged)}`, + }); + expect(callback.statusCode).toBeGreaterThanOrEqual(400); + expect(calls).toEqual([]); + expect(await store.findStoredLinkedIdentity("joystick", "chan-forged")).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("starts a streampixels connect at joystick.tv with the five scopes and the callback redirect", async () => { const app = await buildApp({ config: createTestConfig() }); apps.push(app); @@ -2096,10 +2140,19 @@ describe("callbackMint", () => { ["discord", "connect", "refused"], ["discord", "sign_in", "refused"], ] as const)("%s %s on streampixels is %s", (provider, mode, expected) => { - expect(callbackMint(streampixels, provider, mode)).toBe(expected); + expect(callbackMint(streampixels, provider, mode, { kind: "creator", targetId: "creator-1" })).toBe(expected); + }); + + it.each([ + ["null", null], + ["undefined", undefined], + ["an account binding", { kind: "account", targetId: "a-1" }], + ["an empty creator targetId", { kind: "creator", targetId: "" }], + ] as const)("a joystick connect with %s is refused", (_name, connection) => { + expect(callbackMint(streampixels, "joystick", "connect", connection)).toBe("refused"); }); it("keeps Bifrost's discord connect a viewer session", () => { - expect(callbackMint(builtInAppProfiles.bifrost!, "discord", "connect")).toBe("viewer_session"); + expect(callbackMint(builtInAppProfiles.bifrost!, "discord", "connect", null)).toBe("viewer_session"); }); }); diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index da7f991..4cd7396 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -562,7 +562,7 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag async function flow(rig: Rig, provider: "joystick" | "twitch" | "youtube", mode: "connect" | "sign_in") { const app = rig.apps[0]!; - const start = await app.inject({ method: "POST", url: `/v1/oauth/${provider}/start`, payload: { appSlug: "streampixels", mode, returnTo } }); + const start = await app.inject({ method: "POST", url: `/v1/oauth/${provider}/start`, payload: { appSlug: "streampixels", mode, returnTo, ...(mode === "connect" ? { connection: { kind: "creator", targetId: "creator-1" } } : {}) } }); expect(start.statusCode).toBe(201); const callback = await app.inject({ method: "GET", From 09ea233592eabf89bc94e12f22b2ca92e863c32f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:34:43 +0200 Subject: [PATCH 69/85] Token answers: an expires_in beyond a year is unusable, so the default lifetime applies A value like 3e11 overflowed the Date the expiry is stored as and surfaced as an untyped error. usableExpiresIn now caps a usable value at 365 days; the refresh path is pinned on memory and live Postgres. Co-Authored-By: Claude Opus 5.5 --- src/oauth.ts | 13 +++++++++++-- tests/managed-credential.test.ts | 21 +++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/oauth.ts b/src/oauth.ts index 2ad2c76..e5411ff 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -228,10 +228,19 @@ export async function requestProviderTokens( return tokenSet; } -/** A positive finite number, or a string that is a positive integer. */ +/** The longest access token lifetime taken at its word: a year. Anything above is not a lifetime. */ +const MAX_USABLE_EXPIRES_IN_SECONDS = 365 * 24 * 60 * 60; + +/** + * A positive finite number of seconds, or a string that is a positive integer, + * no longer than a year. A larger value would overflow the Date the expiry is + * stored as, so it is unusable and the descriptor's default lifetime applies. + */ function usableExpiresIn(value: unknown): number | undefined { const seconds = typeof value === "string" && /^\s*\d+\s*$/.test(value) ? Number(value) : value; - return typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0 ? seconds : undefined; + return typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0 && seconds <= MAX_USABLE_EXPIRES_IN_SECONDS + ? seconds + : undefined; } function sharedRuntimeExchange(provider: Provider): OAuthProviderRuntime["exchangeAuthorizationCode"] { diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 4cd7396..8a9b8de 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -65,10 +65,23 @@ type Mode = | "no_expires_in" | "unusable_expires_in" | "string_expires_in" + | "huge_expires_in" + | "huge_string_expires_in" | "hang"; function expiresInFor(mode: Mode): unknown { - return mode === "unusable_expires_in" ? "soon" : mode === "string_expires_in" ? "7200" : 3600; + switch (mode) { + case "unusable_expires_in": + return "soon"; + case "string_expires_in": + return "7200"; + case "huge_expires_in": + return 3e11; + case "huge_string_expires_in": + return "300000000000"; + default: + return 3600; + } } /** @@ -122,7 +135,9 @@ function installTokenEndpoint(provider: Provider, firstRefreshToken: string) { case "normal": case "no_expires_in": case "unusable_expires_in": - case "string_expires_in": { + case "string_expires_in": + case "huge_expires_in": + case "huge_string_expires_in": { if (state.installationRevoked || presented !== state.live) { state.installationRevoked = true; return new Response(JSON.stringify({ error: "invalid_grant", error_description: "reused CANARY" }), { status: 400 }); @@ -487,6 +502,8 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag ["no expires_in", "no_expires_in", 3600], ["an unusable expires_in", "unusable_expires_in", 3600], ["a numeric-string expires_in", "string_expires_in", 7200], + ["a numeric expires_in beyond a year", "huge_expires_in", 3600], + ["a numeric-string expires_in beyond a year", "huge_string_expires_in", 3600], ] as const)( "refresh-once-per-token: an answer with %s stores now plus %s s, not the old due expiry, and resolve refreshes again only once it passes", timeout, From 4e312828c3cd7918c0ad08ad6754148eb70fb77b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:34:44 +0200 Subject: [PATCH 70/85] Pin the handoff strip: a custody-only backend_callback delivery carries no refresh token Delivers a joystick connect through a backend_callback handoff and checks the posted body has no refreshToken or refresh and empty capabilities, so the handoff spreading `issued` instead of `delivered` fails. Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/app.test.ts b/tests/app.test.ts index 6aed7c6..0a061df 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1409,6 +1409,44 @@ describe("Heimdall service", () => { } }); + it("a backend_callback delivery of a joystick connect carries no refresh token and no capabilities", async () => { + const callbackUrl = "https://streampixels.gamecult.org/api/heimdall/callback"; + const app = await buildApp({ + config: { ...createTestConfig(), appBackendCallbacks: { streampixels: [callbackUrl] } }, + }); + apps.push(app); + stubJoystick({ channel_id: "chan-hand", username: "deru" }); + const providerFetch = globalThis.fetch; + const deliveries: Array> = []; + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + if (String(input) === callbackUrl) { + deliveries.push(JSON.parse(String(init?.body)) as Record); + return new Response(null, { status: 204 }); + } + return providerFetch(input, init); + }) as typeof fetch; + + const start = await connect(app, { + connection: creator, + handoff: { kind: "backend_callback", attemptId: "attempt-js-1", callbackUrl }, + }); + expect(start.statusCode).toBe(201); + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=c&state=${encodeURIComponent(start.json().stateToken as string)}`, + }); + expect(callback.statusCode).toBeLessThan(300); + + expect(deliveries).toHaveLength(1); + const delivered = deliveries[0]!; + expect(delivered).toMatchObject({ status: "success", handoffKind: "backend_callback", mode: "connect", attemptId: "attempt-js-1" }); + expect(typeof delivered.accessToken).toBe("string"); + expect(delivered.refreshToken).toBeUndefined(); + expect(delivered.refresh).toBeUndefined(); + expect(delivered.claimSet.capabilities).toEqual([]); + expect(JSON.stringify(delivered)).not.toContain("js-refresh"); + }); + it("starts a streampixels connect at joystick.tv with the five scopes and the callback redirect", async () => { const app = await buildApp({ config: createTestConfig() }); apps.push(app); From 1d856ebd40118fe82eaf3afa57966ac2c516ffe9 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:35:11 +0200 Subject: [PATCH 71/85] Code exchange: a rotating provider's grant without a refresh token fails the callback Joystick rotates refresh tokens, so a code-exchange answer whose refresh_token is absent or empty would store a credential the first refresh revokes. The shared exchange now throws a typed ProviderResponseError before the identity is resolved or anything is written. The residual-risk comment on the in-flight refresh map cites its follow-up. Co-Authored-By: Claude Opus 5.5 --- src/managed-credential.ts | 6 ++++-- src/oauth.ts | 12 ++++++++++-- tests/managed-credential.test.ts | 33 ++++++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/managed-credential.ts b/src/managed-credential.ts index e2fd124..32e9f51 100644 --- a/src/managed-credential.ts +++ b/src/managed-credential.ts @@ -68,8 +68,10 @@ function withoutTokens(identity: StoredLinkedIdentity): StoredLinkedIdentity { * attempt is in flight join it and receive its outcome, success or failure. * The entry lives exactly as long as the attempt: a later request retries. * - * Accepted residual risk (docs/service-contract.md): one presentation per - * process per burst, so N processes may present one token N times. + * Known residual risk (docs/service-contract.md): one presentation per + * process per burst, so N processes may present one token N times. The + * cross-process failed-refresh marker that would close it is + * joystick:follow_up:cross-process-failed-refresh-marker. */ const refreshesInFlight = new WeakMap>>(); diff --git a/src/oauth.ts b/src/oauth.ts index e5411ff..b6658e8 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -244,12 +244,20 @@ function usableExpiresIn(value: unknown): number | undefined { } function sharedRuntimeExchange(provider: Provider): OAuthProviderRuntime["exchangeAuthorizationCode"] { - return (options) => - requestProviderTokens(provider, options.config, { + return async (options) => { + const tokenSet = await requestProviderTokens(provider, options.config, { grant_type: "authorization_code", code: options.code, redirect_uri: options.redirectUri, }); + // A rotating provider's refresh token is the only way back to the + // credential: a grant that arrives without one would be stored as a row + // the first refresh revokes, so the callback fails before it writes. + if (providerCatalog[provider].rotatesRefreshTokens && !tokenSet.refreshToken) { + throw new ProviderResponseError(200, "provider_body_not_token_set", `${providerCatalog[provider].displayName} token exchange failed`); + } + return tokenSet; + }; } function sharedRuntimeRefresh(provider: Provider): NonNullable { diff --git a/tests/managed-credential.test.ts b/tests/managed-credential.test.ts index 8a9b8de..50ca9ba 100644 --- a/tests/managed-credential.test.ts +++ b/tests/managed-credential.test.ts @@ -561,12 +561,12 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag describe("what the callback mints is decided by mode and provider role", () => { const returnTo = "https://streampixels.gamecult.org/auth/connect"; - function stubProviders(identity: Record) { + function stubProviders(identity: Record, tokenAnswer: Record = { refresh_token: "tok-refresh-CANARY" }) { globalThis.fetch = (async (input: string | URL) => { const url = String(input); const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 }); if (url.startsWith("https://api.joystick.tv/api/oauth/token") || Object.values(providerCatalog).some((d) => d.tokenEndpoint === url)) { - return json({ access_token: "tok-access", refresh_token: "tok-refresh-CANARY", token_type: "Bearer", expires_in: 3600 }); + return json({ access_token: "tok-access", token_type: "Bearer", expires_in: 3600, ...tokenAnswer }); } if (url === "https://api.joystick.tv/api/v1/me/identity") return json({ channel_id: identity.joystick, username: "deru" }); if (url === "https://api.twitch.tv/helix/users") return json({ data: [{ id: identity.twitch, login: "deru", display_name: "Deru" }] }); @@ -618,6 +618,35 @@ for (const { kind, live } of rigKinds) describe.skipIf(live && !adminUrl)(`manag expect(rig.custody.decrypt(row.refreshTokenEncrypted!)).toBe("tok-refresh-CANARY"); }); + it.each([ + ["no refresh_token", {}], + ["an empty refresh_token", { refresh_token: "" }], + ])("a rotating provider's code exchange with %s fails the callback before anything is written", timeout, async (_name, tokenAnswer) => { + const rig = await makeRig("joystick"); + const before = await rig.row(); + const auditBefore = (await rig.audit()).length; + stubProviders({ joystick: "channel-new", twitch: "t-1", youtube: "y-1" }, tokenAnswer); + const app = rig.apps[0]!; + const start = await app.inject({ + method: "POST", + url: "/v1/oauth/joystick/start", + payload: { appSlug: "streampixels", mode: "connect", returnTo, connection: { kind: "creator", targetId: "creator-1" } }, + }); + expect(start.statusCode).toBe(201); + const callback = await app.inject({ + method: "GET", + url: `/v1/oauth/joystick/callback?code=c&state=${encodeURIComponent(start.json().stateToken as string)}`, + }); + expect(callback.statusCode).toBeGreaterThanOrEqual(400); + expect(callback.body).not.toContain("tok-access"); + expect(await rig.row()).toEqual(before); + expect(await rig.stores[0]!.findStoredLinkedIdentity("joystick", "channel-new")).toBeNull(); + const failures = (await rig.audit()).slice(auditBefore); + expect(failures).toEqual([ + { eventType: "oauth_callback_failed", payload: expect.objectContaining({ errorClass: "ProviderResponseError", providerStatus: 200 }) }, + ]); + }); + it.each(["twitch", "youtube"] as const)("a %s sign_in on streampixels still yields viewer_access and a refresh token", timeout, async (provider) => { const rig = await makeRig("joystick"); stubProviders({ joystick: "channel-1", twitch: "t-1", youtube: "y-1" }); From 6db84bfae8a9bfa29b7a8bf09c25f6889b13b4aa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 14:45:35 +0200 Subject: [PATCH 72/85] Pin the year boundary, the typed exchange failure and the start refusal's code Closes StrykerJS survivors on the diff: the cap at exactly 365 days against a second over, the rotating-exchange failure's typed code (and that a non-rotating provider is unaffected), and the start refusal's error word. Co-Authored-By: Claude Opus 5.5 --- tests/app.test.ts | 9 +++++---- tests/provider-token-request.test.ts | 29 +++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/app.test.ts b/tests/app.test.ts index 0a061df..7f4d96d 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -1367,14 +1367,15 @@ describe("Heimdall service", () => { } it.each([ - ["no connection", {}], - ["an account connection", { connection: { kind: "account", targetId: "victim-1" } }], - ["an empty targetId", { connection: { kind: "creator", targetId: "" } }], - ])("refuses a joystick connect with %s at start, before any redirect", async (_name, extra) => { + ["no connection", {}, "creator_connection_required"], + ["an account connection", { connection: { kind: "account", targetId: "victim-1" } }, "creator_connection_required"], + ["an empty targetId", { connection: { kind: "creator", targetId: "" } }, "invalid_request"], + ])("refuses a joystick connect with %s at start, before any redirect", async (_name, extra, error) => { const app = await buildApp({ config: createTestConfig() }); apps.push(app); const start = await connect(app, extra); expect(start.statusCode).toBe(400); + expect(start.json().error).toBe(error); expect(start.json().authorizationUrl).toBeUndefined(); expect(start.json().stateToken).toBeUndefined(); }); diff --git a/tests/provider-token-request.test.ts b/tests/provider-token-request.test.ts index ff415fd..6a778f5 100644 --- a/tests/provider-token-request.test.ts +++ b/tests/provider-token-request.test.ts @@ -2,7 +2,7 @@ import { inspect } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { type HeimdallConfig } from "../src/config.js"; import { providers, type Provider } from "../src/contracts.js"; -import { requestProviderTokens } from "../src/oauth.js"; +import { createOAuthRuntimeRegistry, requestProviderTokens } from "../src/oauth.js"; import { providerCatalog } from "../src/providers.js"; const originalFetch = globalThis.fetch; @@ -138,6 +138,33 @@ describe("requestProviderTokens follows the provider descriptor", () => { expect(Date.parse(tokens.expiresAt ?? "") - before).toBeLessThan(3600 * 1000 + 5_000); }); + it.each([ + ["exactly a year", 31_536_000, 31_536_000], + ["a second over a year", 31_536_001, 3600], + ["3e11", 3e11, 3600], + ["3e11 as a string", "300000000000", 3600], + ])("takes an expires_in of %s as %i seconds' worth of lifetime", async (_n, expiresIn, seconds) => { + capture(() => new Response(JSON.stringify({ access_token: "a", expires_in: expiresIn }), { status: 200 })); + const before = Date.now(); + const tokens = await requestProviderTokens("twitch", configFor(), { grant_type: "refresh_token", refresh_token: "R" }); + expect(Date.parse(tokens.expiresAt ?? "") - before).toBeGreaterThanOrEqual(seconds * 1000 - 1000); + expect(Date.parse(tokens.expiresAt ?? "") - before).toBeLessThan(seconds * 1000 + 5_000); + }); + + it.each([ + ["no refresh_token", {}], + ["an empty refresh_token", { refresh_token: "" }], + ])("a rotating provider's code exchange with %s is a typed malformed token set; a non-rotating one is not", async (_n, extra) => { + capture(() => new Response(JSON.stringify({ access_token: "a", expires_in: 3600, ...extra }), { status: 200 })); + const runtimes = createOAuthRuntimeRegistry(); + const exchange = { config: configFor(), code: "c", redirectUri: "https://h/cb" }; + await expect(runtimes.joystick.exchangeAuthorizationCode(exchange)).rejects.toMatchObject({ + name: "ProviderResponseError", + code: "provider_body_not_token_set", + }); + expect((await runtimes.twitch.exchangeAuthorizationCode(exchange)).refreshToken).toBeUndefined(); + }); + it.each([ ["a number", 42], ["an object", { a: 1 }], From 778eabaedc9b662dfd13ee93d00bbc4cd459bfcc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 15:49:34 +0200 Subject: [PATCH 73/85] Docs: map managed credential custody, the callback mint and the open follow-ups architecture.md gains a Managed credential custody section: the single refresh owner, the store-port lock, per-process single flight, typed revoked and refresh-failed answers, the provider descriptor token fields and the one token request, the mode x role gates, custody-only completions, verification layer and follow-ups by id. The spotify finding now matches the removed spiverse profile; JOYSTICK joins the provider env lists. implementation-plan and service-contract lose two stale lines. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 184 ++++++++++++++++++++++++++++++++---- docs/implementation-plan.md | 4 +- docs/service-contract.md | 2 +- 3 files changed, 168 insertions(+), 22 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 88bfeaa..8ae5c19 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -305,6 +305,159 @@ Important rule: connection is tied to a user or account rather than a purely app-local secret - app-specific binding semantics and runtime behavior still stay app-owned +## Managed credential custody + +Source-grounded as of branch `eureka/joystick-h1` at 6db84bf (PR +GameCult/Heimdall#1, unmerged). Nothing here is deployed. + +### One owner: `refreshManagedCredential` + +`src/managed-credential.ts` owns every decision to present a stored refresh +token to a provider. The entitlement path (`refreshAppSession`), the Bifrost +Patreon sync and `managed-credentials/resolve` all call it; none of them reads +`tokenExpiresAt` or a provider token endpoint itself. The earlier +`refreshLinkedIdentityCredentialIfNeeded` is deleted, and callers no longer +write refreshed tokens through `upsertLinkedIdentity`: a refresh reaches the row +only through the store port below. The only other writer of credential columns +is the OAuth callback's grant write (`app.ts`, once). + +| | | +|---|---| +| Input | a `StoredLinkedIdentity` snapshot, the config, the token custody (AES-256-GCM seal), the provider runtimes, an injectable clock and timeout | +| Output | `{kind: "current" or "revoked", identity}`, or a thrown `ManagedCredentialRefreshFailed` (carries the provider and nothing the provider said) | +| Forbidden writers | any caller writing token columns or `token_expires_at` outside `updateLinkedIdentityCredential` and the callback's grant write; any caller computing "is a refresh due" | + +The snapshot is a hint. It may answer without a lock when the identity holds no +access token (`revoked`) or no refresh is due (`current`; due means a known +expiry within 5 minutes, a stored refresh token, and a runtime that refreshes). +It never decides that a refresh happens. + +### Three nested guards + +1. **Single flight, per process.** `refreshesInFlight` is keyed by store, then + linked identity id. A caller arriving while an attempt is in flight joins it + and receives its outcome, success or failure, so nobody queues behind a + failed holder and re-presents the same token. The entry ends with the + attempt; a later request retries. +2. **Store-port lock, across processes.** + `HeimdallStore.updateLinkedIdentityCredential(id, decide)` runs `decide` + against the row as read under the lock. `PostgresStore` takes + `SELECT ... FOR UPDATE` inside a transaction; `InMemoryStore` serialises per + id on a promise chain that deletes its entry only when it is still the tail. + `decide` re-checks `dueForRefresh` on the locked row, so a process that + waited finds the token already rotated and returns `keep`. The provider call + is aborted at 10 seconds (`MANAGED_CREDENTIAL_REFRESH_TIMEOUT_MS`), which + bounds how long any waiter holds a pooled connection. +3. **Decision, not mutation.** `decide` returns a `CredentialDecision`: `keep`, + `revoke`, or `rotate` with the access token, refresh token and + `tokenExpiresAt` all required. The store applies it and returns the row. The + new pair is persisted before the access token is returned. A store that + keeps the old expiry on rotate is a type error, not a test. + +### Typed answers + +- **Revoked** (`410 managed_credential_revoked`, audit + `managed_credential_revoked {provider, reason}`, all three token columns + cleared). Two reasons only: `provider_revoked`, an HTTP 400 whose OAuth + `error` is exactly `invalid_grant`; and `rotation_missing`, a well-formed + refresh answer without a refresh token from a provider whose descriptor says + `rotatesRefreshTokens`. An identity with no access token is also `revoked`. + `404` means no linked identity exists. On the entitlement path a revoked + identity is skipped; on the Bifrost sync it is `404 + patreon_identity_not_linked` with Bifrost uncalled. +- **Refresh failed** (`502 managed_credential_refresh_failed`, tokens + untouched, retry allowed): invalid client, a 401 or 5xx whatever its body, + timeout, a 2xx without a string `access_token` or carrying `error`, a + `refresh_token` present but not a string, an empty or non-JSON body. + `ProviderResponseError.oauthError` is drawn only from the closed RFC 6749 code + set and is read only from a 400; no provider body reaches a message, stack or + audit payload. +- An empty-string `refresh_token` counts as missing. + +### Provider descriptor and the one token request + +`providerCatalog` (`src/providers.ts`) is the only place a provider's token +behaviour is written down. Fields beyond the authorize side: + +- `tokenEndpoint`: where both grants go. +- `tokenEndpointAuthMethod`: `client_secret_basic` or `client_secret_post` + (RFC 7591 names); exactly one client-auth method per request. +- `rotatesRefreshTokens`: true only for Joystick. Drives fail-closed on + refresh (`rotation_missing`) and a failed callback on exchange when the grant + carries no refresh token. +- `defaultAccessTokenLifetimeSeconds`: 3600 for all seven providers. Applied + when an answer has no usable `expires_in`; unusable means absent, non-numeric, + not a positive number or positive-integer string, or above 365 days. An + unknown expiry is therefore never stored, so every credential comes due. + +`requestProviderTokens(provider, config, grant)` in `src/oauth.ts` replaces the +ten per-provider exchange and refresh functions. Authorization-code and refresh +grants share it, it returns an `ExpiringTokenSet` (`expiresAt` required), and it +rejects a malformed 2xx typed as `provider_body_not_token_set` on both grants. +The per-provider runtimes keep only identity resolution. + +### Mode x role at the gates + +`src/app-profiles.ts` holds both questions, and the profile's role lists are +their only input: + +- `admitsProviderForMode(profile, provider, mode)`, the start gate: `sign_in` + and `link` take `identityProviders` only; `connect` takes + `managedConnectionProviders` or `identityProviders` (Bifrost connects its + identity providers). `supportsProvider` is deleted. +- `callbackMint(profile, provider, mode, connection)`, decided at the callback + from the signed state and asked again at start, so an unbound connect never + reaches a redirect: + - `viewer_session`: the provider is in `identityProviders`; a normal session + with the profile's capabilities. + - `custody_only`: a `connect` for a provider that is only a managed + connection (Joystick on StreamPixels) with `connection.kind = "creator"` and + a non-empty `targetId`. + - `refused`: everything else, before the code exchange. + +A custody-only completion stores the credential and issues a claim with no +facts, no capabilities and no hybrid capabilities (`withoutCapabilities` in +`src/claims.ts`, the one owner of a capability-free claim). The completion +payload and the `backend_callback` handoff carry no `refreshToken` or +`refresh`, because the refresh route recomputes facts from the account and would +turn that session into a viewer session. A session row still exists, and +`BackendHandoffSuccessPayload` types both fields optional for this reason. The +consumer's redeem reads only account id, provider, mode, connection and claim +identities, so the claim form is unchanged for it. + +### Verification layer + +The rules above are pinned by `tests/managed-credential.test.ts` on three rigs +(memory, Postgres with one app, Postgres with two pools), +`tests/provider-token-request.test.ts`, `tests/app.test.ts` (a matrix over all +profiles x providers x modes, callback mint, handoff strip) and +`tests/recipe-secrets.test.ts`. Evidence is the Yggdrasil `ygg-verify` run on +Linux (node:24, postgres:16, `HEIMDALL_REQUIRE_LIVE_PG=1`; 600 passed, 8 +skipped, none a live-Postgres suite), hand and StrykerJS mutants over the diff, +and Soul's pass 4 verdict `joystick:verdict:cut-h1.s4`. The deploy path's own +builder was not used. + +### Open follow-ups + +- `joystick:follow_up:cross-process-failed-refresh-marker`: single flight is per + process, so N Heimdall processes can present one token N times after a + transient failure. Needs a failed-attempt marker on the row, read under the + lock. Must land before or with Idunn onboarding. +- `joystick:follow_up:connect-start-needs-app-auth`: connect starts are + unauthenticated, so anyone can name `{creator, targetId}`; the callback + re-check proves a binding exists, not that the caller owns it. It also owns + the whitespace-only `targetId` that passes both gates and the callback's + fail-open `''` fallback for a vanished profile (finding + `joystick:finding:cut-h1.s4.whitespace-targetid-passes-custody-gate`). Must + land before the public Joystick bot or a second creator. +- `joystick:follow_up:heimdall-idunn-onboarding` (gamecult-ops): live Heimdall + is a hand-copied release under `heimdall.service`, not an Idunn-admitted + generation; the recipe's Joystick lines are declared but unproven by a + deploy. +- `joystick:follow_up:joystick-viewer-signin-public-bot`: Joystick stays out of + StreamPixels `identityProviders` until the public bot is registered; viewer + sign-in must not rewrite a creator's credential row. + ## Policy model The shared authority should not hard-code one app's access rules. It should @@ -514,24 +667,17 @@ each provider**. That URL cannot move on deploy. ### Finding: the `spotify` provider outlived its consumer -`spotiverse` is the only profile naming `spotify`, and Spotiverse was lost with -the `E:` drive — it was a small daemon exposing the Spotify API to agents, and -it no longer exists in the swarm. +`spotiverse` was the only profile naming `spotify`, and Spotiverse was lost with +the `E:` drive. The profile has since been removed from the registry +(`docs/app-profiles/spotiverse.md` is the restore spec), and +`admitsProviderForMode` refuses a provider no profile lists, so +`/v1/oauth/spotify/start` no longer reaches a handshake. -What survives it: `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID` and -`GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET` are configured in -`/srv/heimdall/env/service.env`, the profile is advertised in the public -discovery document, and `/v1/oauth/spotify/start` is reachable from the open -web. That path would run a real OAuth handshake against a real client secret and -take custody of a user's Spotify refresh token on behalf of a consumer that -cannot use it. - -This is not an active compromise. It is a live credential and a token-custody -code path kept alive for nothing, which is exactly the shape that becomes an -incident later. Retire the profile and the provider config together, or record -why Spotiverse is expected back. Retiring is a change to `src/app-profiles.ts` -and the provider catalog and should be decided on its own, not folded into the -boundary change. +What survives: the `spotify` catalog entry and runtime, and, wherever an +operator env still holds them, `GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_ID` and +`GC_ACCESS_PROVIDER_SPOTIFY_CLIENT_SECRET_FILE`. A live credential kept for a +consumer that does not exist is the shape that becomes an incident later. +Retire the provider config, or record why Spotiverse is expected back. ## Deployment modes @@ -588,14 +734,14 @@ Plain settings: - `GC_ACCESS_SIGNING_KEY_ID=...` - `GC_ACCESS_SESSION_TTL_SECONDS=3600` - `GC_ACCESS_STATE_TTL_SECONDS=600` -- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,GITHUB,TWITCH,YOUTUBE,SPOTIFY}_CLIENT_ID=...` +- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,GITHUB,TWITCH,YOUTUBE,SPOTIFY,JOYSTICK}_CLIENT_ID=...` - `GC_ACCESS_APP_{APP_SLUG}_BACKEND_CALLBACK_URLS=...` Secret files: - `GC_ACCESS_DATABASE_URL_FILE=/etc/gamecult/heimdall/credentials/GC_ACCESS_DATABASE_URL` - `GC_ACCESS_TOKEN_ENCRYPTION_KEY_BASE64_FILE=...` -- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,GITHUB,TWITCH,YOUTUBE,SPOTIFY}_CLIENT_SECRET_FILE=...` +- `GC_ACCESS_PROVIDER_{DISCORD,PATREON,GITHUB,TWITCH,YOUTUBE,SPOTIFY,JOYSTICK}_CLIENT_SECRET_FILE=...` - `GC_ACCESS_APP_{APP_SLUG}_SHARED_SECRET_FILE=...` - `GC_ACCESS_BIFROST_PATRON_SUPPORT_SECRET_FILE=...` diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 9a1f84d..44bcbc7 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -44,7 +44,7 @@ What exists right now is: and managed creator connections - an app-authenticated managed-credential resolve endpoint so StreamPixels can call provider APIs without storing refresh tokens locally -- managed Twitch/YouTube credentials refresh inside Heimdall before resolution +- managed Twitch/YouTube/Joystick credentials refresh inside Heimdall before resolution when they are expired or near expiry - StreamPixels now starts viewer and creator Twitch/YouTube OAuth through Heimdall, redeems one-time completion codes server-side, and keeps local @@ -215,7 +215,7 @@ Landed in the first pass: Still to do in this phase: - key rotation policy instead of just stable key loading -- token custody retrieval/refresh surfaces beyond initial storage +- token custody retrieval and refresh have landed (see `docs/architecture.md`, Managed credential custody); the open pieces are the follow-ups listed there ### Phase 2: Heimdall service skeleton diff --git a/docs/service-contract.md b/docs/service-contract.md index fdcc48f..1567ae5 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -173,7 +173,7 @@ Failure modes: Launch note: - Discord and Patreon are the implemented provider runtimes for Repixelizer -- Twitch and YouTube now have generic OAuth exchange and identity runtimes for +- Twitch, YouTube and Joystick have generic OAuth exchange and identity runtimes for StreamPixels viewer auth and managed creator connections - GitHub is still catalogued for start URLs, but its callback runtime is not implemented yet From df05570ef64394b519bebc134e0c472424ebc438 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:56:25 +0200 Subject: [PATCH 74/85] State: Joystick H1 merged and live at d51fa6b; follow-ups recorded by id Co-Authored-By: Claude Opus 5.5 --- notes/fresh-workspace-handoff.md | 26 +++++++++++++++----------- state/evidence.jsonl | 1 + state/map.yaml | 10 +++++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index 4727d7f..ad959f0 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -26,18 +26,23 @@ Do not trust this file for the exact live HEAD. Always check git. ## Active Work -Joystick H1 is the active, M1-critical cut (operator ruling 2026-10-18): -campaign `joystick:campaign:self`, target `joystick:target:r1`, branch -`eureka/joystick-h1`. Read the campaign and target in the typed mind; do not -re-derive them here. The Ghostlight KLTST canaries below stay open, behind H1. +Joystick H1 is merged and live (PR GameCult/Heimdall#1; campaign +`joystick:campaign:self`). Next: `joystick:follow_up:h1-live-round-trips`, then +StreamPixels S0/S1 (other sessions). Open Heimdall follow-ups, by id: +`joystick:follow_up:cross-process-failed-refresh-marker`, +`joystick:follow_up:connect-start-needs-app-auth`, +`joystick:follow_up:heimdall-idunn-onboarding`, +`joystick:follow_up:joystick-viewer-signin-public-bot`. The Ghostlight KLTST +canaries below stay open, behind them. ## Current Orientation -Deploy facts below are last-recorded, from 2026-08-22 (`1086aee`); HEAD has -moved ~100 commits past it. Check Yggdrasil, not this file, for what runs. +Yggdrasil runs Heimdall `d51fa6b`, hand-deployed 2026-10-19 15:51Z; +gamecult-ops `210a1af` records backup, rollback and provenance. Facts below +about Odin, Ghostlight and CultLib pins were verified at `1086aee` (2026-08-22) +and are not re-verified at `d51fa6b`. -- Yggdrasil runs Heimdall - `1086aee01169bf60e8a492b2740db1c6f3e8cabf` with CultLib +- At `1086aee01169bf60e8a492b2740db1c6f3e8cabf` Heimdall ran with CultLib `5cefa0db0079a8e3ee22f29d7b9e6e5aa60912a9` - Heimdall publishes four redacted access records to Odin under unique keys: `eve:provider:heimdall`, `heimdall:command-boundary`, @@ -157,8 +162,7 @@ moved ~100 commits past it. Check Yggdrasil, not this file, for what runs. over `cultnet.transport.rudp.v0` when `GC_ACCESS_IDUNN_RUDP_HEALTH` is configured; HTTP `/healthz`, JWKS, discovery, systemd, and nginx routing remain compatibility witnesses -- Yggdrasil now runs Heimdall commit - `1086aee01169bf60e8a492b2740db1c6f3e8cabf`. Heimdall persists its private +- Since `1086aee` Heimdall persists its private provider-health identity at `/srv/heimdall/keys/provider-health-identity.cc` and publishes canonical signed health independently of the slower Verse snapshot cadence. Idunn owns the exact root trust binding, authenticated @@ -180,7 +184,7 @@ moved ~100 commits past it. Check Yggdrasil, not this file, for what runs. ## Next Real Move -Joystick H1 first (see Active Work). Everything below is the standing queue. +The Active Work follow-ups first. Everything below is the standing queue. Do not continue implementation automatically from a rehydrate-only request. diff --git a/state/evidence.jsonl b/state/evidence.jsonl index 3178947..ba9a942 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -35,3 +35,4 @@ {"ts":"2026-08-22T02:01:00+00:00","type":"deployment","status":"accepted","note":"Yggdrasil runs Heimdall 1086aee01169bf60e8a492b2740db1c6f3e8cabf with CultLib 5cefa0db0079a8e3ee22f29d7b9e6e5aa60912a9. Odin b4f9a2e95f0b41cebdeddc49223781d1d3c7b42a returns Heimdall's provider, command boundary, Eve plugin, and transport profile under unique catalog keys. Ghostlight b515ca90c25573005a616244143803b37f2d06ec accepted canonical auth.begin through the Odin-resolved boundary; only advertised Discord and Heimdall navigation origins reached the browser."} {"ts": "2026-09-06T22:12:03+00:00", "type": "design", "status": "accepted", "note": "App slugs are no longer compile-time. AppSlug is a string; membership is a registry lookup, or isAppSlug shape validation where no store is reachable (the offline verifier). Capability rules are data (anyOf over Heimdall's own fact vocabulary), replacing a per-profile evaluateSharedCapabilities function that could not be accepted from a stranger. POST /v1/apps registers in RFC 7591 shape, gated on GC_ACCESS_APP_REGISTRATION_SECRET and closed until it is set. Built-in slugs are reserved so a registration cannot shadow a shipped app's audience. Motivation: a creator self-hosting Erycina alongside Heimdall would otherwise maintain a patched auth server to add one slug."} {"ts": "2026-09-08T12:20:12+00:00", "type": "security", "status": "accepted", "note": "epiphany/heimdall-caller-identity, second Hands pass: closed the two Soul-proven holes. (1) auth-completions redeem code is now always store-minted (never handoff.attemptId); attemptId survives only as a separate correlation field on StoredAuthCompletion/CreateAuthCompletionInput, and a new store.consumeAuthCompletionByAttempt(appSlug, attemptId, at) lets the private command plane's completeAuth redeem by the handle it already owns instead of forging an HTTP self-call with completionCode=handle. (2) startOAuthFlow now refuses any returnTo whose origin is not in AppProfile.allowedReturnOrigins (repixelizer/streampixels/bifrost -> .gamecult.org, ghostlight -> yggdrasil.gamecult.org per its docs/app-profiles doc and existing test fixtures; a registered app's allowlist is derived from its stored redirectUris origins). AppCaller is now branded with an unexported Symbol (app-caller.ts); private-command-plane.ts's two { appSlug } literals are gone, replaced by app-caller.ts's callerFromOpenedEnvelope. Deleted: validateAppRegistration, AppRegistrationRequest, RegistrationProblem, AppRegistrationError, HeimdallStore.registerApp (writer). Kept: findRegisteredApp/listRegisteredApps/resolveAppProfile fallback -- AppSlug is a plain string (contracts.ts), registered_apps is a real extension point for a slug Heimdall was never rebuilt for, and isBuiltInApp's only production call site (listAppProfiles) depends on that fallback existing; a row is now provisioned by direct SQL (or InMemoryStore.seedRegisteredApp in tests), not by any app-facing writer. Deleted the two appSlug===\"ghostlight\" branches in private-command-plane.ts; the requirement is now AppProfile.requiredEntitlementPolicyKind, checked generically through resolveAppProfile. tsc --noEmit clean; vitest 94 passed/1 pre-existing-skip, only pre-existing failure is tests/odin-publication.test.ts (Cannot find module 'cultcache-ts', present on main too).", "branch": "epiphany/heimdall-caller-identity"} +{"ts": "2026-10-19T15:56:24+00:00", "type": "design", "status": "accepted", "note": "Refresh is now one locked primitive for every provider (replaces unserialized refreshLinkedIdentityCredentialIfNeeded); Joystick is managed-only (Heimdall-held credential, no viewer sign-in until joystick:follow_up:joystick-viewer-signin-public-bot). Merged in PR #1, live at d51fa6b; campaign joystick:campaign:self.", "branch": "eureka/joystick-h1"} diff --git a/state/map.yaml b/state/map.yaml index 3f8772a..413718e 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -59,8 +59,8 @@ data_flow: - reject per-request auth round-trip chatter active_subgoals: - - ACTIVE, M1-critical (operator ruling 2026-10-18): Joystick H1, campaign joystick:campaign:self (branch eureka/joystick-h1). Joystick OAuth provider, joystick as a streampixels viewer provider, one locked refresh primitive for every provider replacing refreshLinkedIdentityCredentialIfNeeded. - - OPEN, behind H1: pressure the deployed Ghostlight gamecult.heimdall.access integration with two- and eight-account human canaries. + - Joystick H1 is merged and live (PR GameCult/Heimdall#1, d51fa6b; campaign joystick:campaign:self). Open follow-ups by id: joystick:follow_up:h1-live-round-trips (next), cross-process-failed-refresh-marker, connect-start-needs-app-auth, heimdall-idunn-onboarding, joystick-viewer-signin-public-bot. StreamPixels S0/S1 run in other sessions. + - OPEN, behind the H1 live round-trips: pressure the deployed Ghostlight gamecult.heimdall.access integration with two- and eight-account human canaries. - Keep the shared architecture explicit and separate from app profiles. - Preserve the default boundary that app-domain data remains app-owned. - Align the plan to Heimdall as a Yggdrasil-hosted authority service. @@ -95,7 +95,7 @@ accepted_design: landed_baseline: - Heimdall publishes its redacted provider, command boundary, Eve plugin, and transport profile through Odin under globally unique record keys; Ghostlight resolves that boundary instead of owning a direct endpoint. - - Yggdrasil runs Heimdall 1086aee01169bf60e8a492b2740db1c6f3e8cabf with CultLib 5cefa0db0079a8e3ee22f29d7b9e6e5aa60912a9; live Ghostlight auth.begin through Odin discovery is accepted. + - Yggdrasil runs Heimdall d51fa6b (hand-deployed 2026-10-19 15:51Z; gamecult-ops 210a1af holds backup, rollback and provenance; prior 1086aee) with CultLib 5cefa0db0079a8e3ee22f29d7b9e6e5aa60912a9; live Ghostlight auth.begin through Odin discovery is accepted. - The Heimdall access plugin, persistent OAuth attempts, encrypted private CultNet commands, conformance fixtures, and browser reference adapter are implemented and locally verified. - Exact-session refresh validation and logout revocation are implemented; stale or racing refresh cannot resurrect a revoked session. - Shared architecture extracted into its own repo and renamed to Heimdall. @@ -126,5 +126,5 @@ rejected_paths: reason: reusing auth mechanics does not justify centralizing app-domain audience or product data current_status: - summary: Active work is Joystick H1 (campaign joystick:campaign:self). Last recorded Yggdrasil deploy (not verifiable from this repo; HEAD is far past it, see git): Yggdrasil runs the gamecult.heimdall.access plugin and encrypted loopback begin, complete, refresh, and logout plane at 1086aee01169bf60e8a492b2740db1c6f3e8cabf. Odin returns all four redacted discovery records, Ghostlight resolves the boundary through Odin, and live anonymous auth.begin is accepted without exposing claims or secrets. - next_action: Land Joystick H1 per campaign joystick:campaign:self and target joystick:target:r1 (worktree and branch eureka/joystick-h1). Then, still open, run separate-account KLTST completion, refresh, logout, privacy, and stale-attempt canaries through the deployed Ghostlight Eve host; keep provider credential rotation on the operator list because prior tool output exposed OAuth secrets. + summary: Yggdrasil runs Heimdall release d51fa6b (hand-deployed 2026-10-19 15:51Z, gamecult-ops 210a1af), which includes Joystick H1 (PR #1). The plugin and encrypted loopback begin, complete, refresh and logout plane, the Odin discovery records and Ghostlight resolution were verified at 1086aee (2026-08-22) and are not re-verified at d51fa6b. + next_action: Run joystick:follow_up:h1-live-round-trips, then StreamPixels S0/S1 (other sessions). Still open behind that: separate-account KLTST completion, refresh, logout, privacy, and stale-attempt canaries through the deployed Ghostlight Eve host; keep provider credential rotation on the operator list because prior tool output exposed OAuth secrets. From 30a1ace6fdfa22a7be25a7ec11a4d012b51bec7f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 17:56:34 +0200 Subject: [PATCH 75/85] State: close eureka/joystick-h1 as accepted (merged, PR #1) Co-Authored-By: Claude Opus 5.5 --- state/branches.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/state/branches.json b/state/branches.json index 643b64f..32cd28f 100644 --- a/state/branches.json +++ b/state/branches.json @@ -16,11 +16,11 @@ { "id": "eureka/joystick-h1", "hypothesis": "One locked refresh primitive plus a Joystick OAuth provider in Heimdall lets StreamPixels ingest Joystick without ever presenting a provider refresh token.", - "status": "active", + "status": "accepted", "artifacts": [ "state/map.yaml" ], - "notes": "Campaign joystick:campaign:self, target joystick:target:r1. Hands owns code in a separate worktree." + "notes": "Merged as PR GameCult/Heimdall#1 (d51fa6b), deployed to Yggdrasil 2026-10-19 15:51Z; gamecult-ops 210a1af. Campaign joystick:campaign:self." } ] } From 5952cce8a0f52691284e34e5530c23c503b6f03b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 18:03:17 +0200 Subject: [PATCH 76/85] State: live d51fa6b proven by a human Twitch sign-in; Joystick consent remains Co-Authored-By: Claude Opus 5.5 --- notes/fresh-workspace-handoff.md | 2 +- state/evidence.jsonl | 1 + state/map.yaml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/notes/fresh-workspace-handoff.md b/notes/fresh-workspace-handoff.md index ad959f0..9d23cf1 100644 --- a/notes/fresh-workspace-handoff.md +++ b/notes/fresh-workspace-handoff.md @@ -27,7 +27,7 @@ Do not trust this file for the exact live HEAD. Always check git. ## Active Work Joystick H1 is merged and live (PR GameCult/Heimdall#1; campaign -`joystick:campaign:self`). Next: `joystick:follow_up:h1-live-round-trips`, then +`joystick:campaign:self`). Next: `joystick:follow_up:h1-joystick-consent-round-trip`, then StreamPixels S0/S1 (other sessions). Open Heimdall follow-ups, by id: `joystick:follow_up:cross-process-failed-refresh-marker`, `joystick:follow_up:connect-start-needs-app-auth`, diff --git a/state/evidence.jsonl b/state/evidence.jsonl index ba9a942..5329d5f 100644 --- a/state/evidence.jsonl +++ b/state/evidence.jsonl @@ -36,3 +36,4 @@ {"ts": "2026-09-06T22:12:03+00:00", "type": "design", "status": "accepted", "note": "App slugs are no longer compile-time. AppSlug is a string; membership is a registry lookup, or isAppSlug shape validation where no store is reachable (the offline verifier). Capability rules are data (anyOf over Heimdall's own fact vocabulary), replacing a per-profile evaluateSharedCapabilities function that could not be accepted from a stranger. POST /v1/apps registers in RFC 7591 shape, gated on GC_ACCESS_APP_REGISTRATION_SECRET and closed until it is set. Built-in slugs are reserved so a registration cannot shadow a shipped app's audience. Motivation: a creator self-hosting Erycina alongside Heimdall would otherwise maintain a patched auth server to add one slug."} {"ts": "2026-09-08T12:20:12+00:00", "type": "security", "status": "accepted", "note": "epiphany/heimdall-caller-identity, second Hands pass: closed the two Soul-proven holes. (1) auth-completions redeem code is now always store-minted (never handoff.attemptId); attemptId survives only as a separate correlation field on StoredAuthCompletion/CreateAuthCompletionInput, and a new store.consumeAuthCompletionByAttempt(appSlug, attemptId, at) lets the private command plane's completeAuth redeem by the handle it already owns instead of forging an HTTP self-call with completionCode=handle. (2) startOAuthFlow now refuses any returnTo whose origin is not in AppProfile.allowedReturnOrigins (repixelizer/streampixels/bifrost -> .gamecult.org, ghostlight -> yggdrasil.gamecult.org per its docs/app-profiles doc and existing test fixtures; a registered app's allowlist is derived from its stored redirectUris origins). AppCaller is now branded with an unexported Symbol (app-caller.ts); private-command-plane.ts's two { appSlug } literals are gone, replaced by app-caller.ts's callerFromOpenedEnvelope. Deleted: validateAppRegistration, AppRegistrationRequest, RegistrationProblem, AppRegistrationError, HeimdallStore.registerApp (writer). Kept: findRegisteredApp/listRegisteredApps/resolveAppProfile fallback -- AppSlug is a plain string (contracts.ts), registered_apps is a real extension point for a slug Heimdall was never rebuilt for, and isBuiltInApp's only production call site (listAppProfiles) depends on that fallback existing; a row is now provisioned by direct SQL (or InMemoryStore.seedRegisteredApp in tests), not by any app-facing writer. Deleted the two appSlug===\"ghostlight\" branches in private-command-plane.ts; the requirement is now AppProfile.requiredEntitlementPolicyKind, checked generically through resolveAppProfile. tsc --noEmit clean; vitest 94 passed/1 pre-existing-skip, only pre-existing failure is tests/odin-publication.test.ts (Cannot find module 'cultcache-ts', present on main too).", "branch": "epiphany/heimdall-caller-identity"} {"ts": "2026-10-19T15:56:24+00:00", "type": "design", "status": "accepted", "note": "Refresh is now one locked primitive for every provider (replaces unserialized refreshLinkedIdentityCredentialIfNeeded); Joystick is managed-only (Heimdall-held credential, no viewer sign-in until joystick:follow_up:joystick-viewer-signin-public-bot). Merged in PR #1, live at d51fa6b; campaign joystick:campaign:self.", "branch": "eureka/joystick-h1"} +{"ts": "2026-10-19T16:03:16+00:00", "type": "verification", "status": "accepted", "note": "Live d51fa6b (H1) verified by a real human sign-in: the operator's StreamPixels Twitch sign-in through Heimdall start/callback, 2026-10-19 ('Twitch auth worked fine'); a streampixels session was created 16:02:33Z, after the 15:51Z switch. Joystick consent round trip still open: joystick:follow_up:h1-joystick-consent-round-trip."} diff --git a/state/map.yaml b/state/map.yaml index 413718e..4aa4d5a 100644 --- a/state/map.yaml +++ b/state/map.yaml @@ -59,7 +59,7 @@ data_flow: - reject per-request auth round-trip chatter active_subgoals: - - Joystick H1 is merged and live (PR GameCult/Heimdall#1, d51fa6b; campaign joystick:campaign:self). Open follow-ups by id: joystick:follow_up:h1-live-round-trips (next), cross-process-failed-refresh-marker, connect-start-needs-app-auth, heimdall-idunn-onboarding, joystick-viewer-signin-public-bot. StreamPixels S0/S1 run in other sessions. + - Joystick H1 is merged and live (PR GameCult/Heimdall#1, d51fa6b; campaign joystick:campaign:self). Open follow-ups by id: joystick:follow_up:h1-joystick-consent-round-trip (next), cross-process-failed-refresh-marker, connect-start-needs-app-auth, heimdall-idunn-onboarding, joystick-viewer-signin-public-bot. StreamPixels S0/S1 run in other sessions. - OPEN, behind the H1 live round-trips: pressure the deployed Ghostlight gamecult.heimdall.access integration with two- and eight-account human canaries. - Keep the shared architecture explicit and separate from app profiles. - Preserve the default boundary that app-domain data remains app-owned. @@ -127,4 +127,4 @@ rejected_paths: current_status: summary: Yggdrasil runs Heimdall release d51fa6b (hand-deployed 2026-10-19 15:51Z, gamecult-ops 210a1af), which includes Joystick H1 (PR #1). The plugin and encrypted loopback begin, complete, refresh and logout plane, the Odin discovery records and Ghostlight resolution were verified at 1086aee (2026-08-22) and are not re-verified at d51fa6b. - next_action: Run joystick:follow_up:h1-live-round-trips, then StreamPixels S0/S1 (other sessions). Still open behind that: separate-account KLTST completion, refresh, logout, privacy, and stale-attempt canaries through the deployed Ghostlight Eve host; keep provider credential rotation on the operator list because prior tool output exposed OAuth secrets. + next_action: Run joystick:follow_up:h1-joystick-consent-round-trip, then StreamPixels S0/S1 (other sessions). Still open behind that: separate-account KLTST completion, refresh, logout, privacy, and stale-attempt canaries through the deployed Ghostlight Eve host; keep provider credential rotation on the operator list because prior tool output exposed OAuth secrets. From c7cf78bbbce303f4e218ae15cbc8826fab840e1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 20:03:50 +0200 Subject: [PATCH 77/85] README: front door with value, problem, entry points, live versus planned Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index a34dc85..b86dbdc 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,19 @@ The point is painfully simple: become "everything phones home for every request" - keep app-domain data separate so "shared auth" does not become "shared swamp" +## Front Door + +- **Value:** one shared auth authority: provider OAuth (Discord, Patreon, GitHub, Twitch, YouTube, Spotify), linked identities, grants, entitlements and session issuance, with Ed25519-signed claims that app backends verify locally. +- **Problem:** every GameCult app was rebuilding Discord/Patreon login glue. Heimdall centralizes identity authority and keeps app-domain data in the apps. +- **Entry points:** + - `src/index.ts` and `src/app.ts`: the Fastify service. `src/signing.ts` and `src/claims.ts` issue JWTs and serve `/.well-known/jwks.json`; `src/oauth.ts` and `src/providers.ts` own provider flows. + - `src/verifier.ts`: the local verifier helper an app backend imports to validate Heimdall JWTs without phoning home. + - `src/odin-publication.ts` and `plugins/gamecult.heimdall.access`: the redacted Odin/Eve provider advertisement and the `gamecult.heimdall.access` plugin (`docs/eve-access-plugin.md`). + - `docs/architecture.md`, `docs/service-contract.md`, `docs/verse-service-contract.md`, `docs/app-profiles/`: the contracts. + - `pnpm install`, `pnpm dev`, `pnpm test` (see Dev Commands). +- **Live:** a stateful first slice: signing and JWKS, OAuth state for six providers, in-memory and Postgres stores, a real Discord callback path for the Repixelizer flow, sealed provider tokens, the loopback CultNet operation boundary, Idunn health publication, and the deployed Ghostlight Eve host consuming the discovered boundary. +- **Planned:** the two-account and eight-account Ghostlight human canaries, refresh and revocation flows, admin surfaces, and broader inspection projections beyond the redacted access boundary. + ## Current Shape This repo now has a stateful first slice instead of a decorative skeleton. From c15fb97706807d9a95837b04c8bfe819ff7d180e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 21:00:46 +0200 Subject: [PATCH 78/85] Test: malformed URL must not reach a sibling prefix's not-found handler (GHSA-p68q-wchp-6fh7) Red on fastify 5.8.5, the live lockfile. Co-Authored-By: Claude Sonnet 5.5 --- tests/fastify-malformed-url.test.ts | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/fastify-malformed-url.test.ts diff --git a/tests/fastify-malformed-url.test.ts b/tests/fastify-malformed-url.test.ts new file mode 100644 index 0000000..f515c74 --- /dev/null +++ b/tests/fastify-malformed-url.test.ts @@ -0,0 +1,47 @@ +import Fastify, { type FastifyReply, type FastifyRequest } from "fastify"; +import { afterEach, describe, expect, it } from "vitest"; + +// GHSA-p68q-wchp-6fh7 (fastify < 5.12.2): a malformed request target under one +// plugin prefix, with a method that has no route in the main router, was +// dispatched to the not-found handler of a sibling plugin, skipping that +// handler's preHandler. This pins the framework fix; it is not about a Heimdall +// route. +describe("fastify malformed URL under sibling not-found handlers", () => { + const apps: Array> = []; + afterEach(async () => { + await Promise.all(apps.splice(0).map((app) => app.close())); + }); + + async function build() { + const app = Fastify(); + apps.push(app); + await app.register(async (pub) => { + pub.get("/ping", async () => "pong"); + }, { prefix: "/public" }); + await app.register(async (priv) => { + priv.setNotFoundHandler( + { + preHandler: async (request: FastifyRequest, reply: FastifyReply) => { + if (!request.headers.authorization) return reply.code(401).send({ error: "auth" }); + }, + }, + (_request: FastifyRequest, reply: FastifyReply) => reply.code(404).send({ secret: "PRIVATE-TENANT-DATA" }), + ); + priv.get("/x", async () => "x"); + }, { prefix: "/private" }); + return app; + } + + it("fails closed with 400 and never reaches another prefix's not-found handler", async () => { + const app = await build(); + const response = await app.inject({ method: "DELETE", url: "/public/%zz" }); + expect(response.body).not.toContain("PRIVATE-TENANT-DATA"); + expect(response.statusCode).toBe(400); + }); + + it("still runs the authenticated not-found preHandler for a well-formed URL", async () => { + const app = await build(); + const response = await app.inject({ method: "DELETE", url: "/private/nothing" }); + expect(response.statusCode).toBe(401); + }); +}); From 899ea0b8fe59dc02ca1b3e66a0ee7f5f2e51c9d2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 22 Oct 2026 21:00:47 +0200 Subject: [PATCH 79/85] Heimdall: fastify 5.8.5 to 5.12.7, find-my-way 9.5.0 to 9.10.0 (clears 5 High advisories) GHSA-p68q-wchp-6fh7 authentication bypass via malformed URLs, header and request validation bypass, request body replacement, find-my-way HTTP/2 DoS. pnpm audit --prod: 8 vulnerabilities before, none after. Co-Authored-By: Claude Sonnet 5.5 --- package.json | 2 +- pnpm-lock.yaml | 47 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 0c6cf4f..36394ef 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "@gamecult/cultcache-ts": "file:vendor/CultLib/packages/cultcache-ts", "cultmesh-ts": "file:vendor/CultLib/packages/cultmesh-ts", "cultnet-ts": "file:vendor/CultLib/packages/cultnet-ts", - "fastify": "^5.8.5", + "fastify": "^5.12.6", "pg": "^8.20.0", "zod": "^3.25.76" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 005bb26..855ad01 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,8 +28,8 @@ importers: specifier: file:vendor/CultLib/packages/cultnet-ts version: file:vendor/CultLib/packages/cultnet-ts fastify: - specifier: ^5.8.5 - version: 5.8.5 + specifier: ^5.12.6 + version: 5.12.7 pg: specifier: ^8.20.0 version: 8.20.0 @@ -532,17 +532,23 @@ packages: fast-json-stringify@6.3.0: resolution: {integrity: sha512-oRCntNDY/329HJPlmdNLIdogNtt6Vyjb1WuT01Soss3slIdyUp8kAcDU3saQTOquEK8KFVfwIIF7FebxUAu+yA==} + fast-json-stringify@7.1.3: + resolution: {integrity: sha512-5TDM1yOF/BDFOclKz2G6rs1RsIl5HXMCrg/npmdNIPNQCezshR5kIHjhwBNuMY67jhkTUAm9DguL8xkL/x0Wtw==} + fast-querystring@1.1.2: resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + fast-uri@4.2.3: + resolution: {integrity: sha512-I1PXwP5eAhlNlcdpjt9CgB59op2ZvZfxw+BkqGFVIUOgNpCVTcTqp3HhTX7brxj5vsip+nSY4nTvff+eIA76hg==} + fastify-plugin@5.1.0: resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==} - fastify@5.8.5: - resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==} + fastify@5.12.7: + resolution: {integrity: sha512-v4pEPsTRPuqlDIYiQFB+BHjVN4SdHX9u0ABk0ZDUKbbdY1vsNryUd1j4Z+jQEZajvYIVk047Kimva9/AJ79GYw==} fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -556,8 +562,8 @@ packages: picomatch: optional: true - find-my-way@9.5.0: - resolution: {integrity: sha512-VW2RfnmscZO5KgBY5XVyKREMW5nMZcxDy+buTOsL+zIPnBlbKm+00sgzoQzq1EVh4aALZLfKdwv6atBGcjvjrQ==} + find-my-way@9.10.0: + resolution: {integrity: sha512-eru+uCPQ2g+nLBkf84MHP/KyNNMar0BDy0j+i8Fb72GF9/YHBZ58q7FODUtfVV7XU9nfRUHXSmaM8NmZcIbZ4Q==} engines: {node: '>=20'} fsevents@2.3.3: @@ -853,8 +859,8 @@ packages: process-warning@4.0.1: resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} - process-warning@5.0.0: - resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + process-warning@5.1.1: + resolution: {integrity: sha512-oCkNVj4OKnKEcocVBcDfiIugN1/cE6rgfT3nCtzeqZPA8+rIxMaQ0xXzIkGP8Gj2qUaP1UR7uhJ1VD8lUCpzCg==} quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} @@ -1475,15 +1481,26 @@ snapshots: json-schema-ref-resolver: 3.0.0 rfdc: 1.4.1 + fast-json-stringify@7.1.3: + dependencies: + '@fastify/merge-json-schemas': 0.2.1 + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + fast-uri: 4.2.3 + json-schema-ref-resolver: 3.0.0 + rfdc: 1.4.1 + fast-querystring@1.1.2: dependencies: fast-decode-uri-component: 1.0.1 fast-uri@3.1.0: {} + fast-uri@4.2.3: {} + fastify-plugin@5.1.0: {} - fastify@5.8.5: + fastify@5.12.7: dependencies: '@fastify/ajv-compiler': 4.0.5 '@fastify/error': 4.2.0 @@ -1491,11 +1508,11 @@ snapshots: '@fastify/proxy-addr': 5.1.0 abstract-logging: 2.0.1 avvio: 9.2.0 - fast-json-stringify: 6.3.0 - find-my-way: 9.5.0 + fast-json-stringify: 7.1.3 + find-my-way: 9.10.0 light-my-request: 6.6.0 pino: 10.3.1 - process-warning: 5.0.0 + process-warning: 5.1.1 rfdc: 1.4.1 secure-json-parse: 4.1.0 semver: 7.7.4 @@ -1509,7 +1526,7 @@ snapshots: optionalDependencies: picomatch: 4.0.4 - find-my-way@9.5.0: + find-my-way@9.10.0: dependencies: fast-deep-equal: 3.1.3 fast-querystring: 1.1.2 @@ -1733,7 +1750,7 @@ snapshots: on-exit-leak-free: 2.1.2 pino-abstract-transport: 3.0.0 pino-std-serializers: 7.1.0 - process-warning: 5.0.0 + process-warning: 5.1.1 quick-format-unescaped: 4.0.4 real-require: 0.2.0 safe-stable-stringify: 2.5.0 @@ -1758,7 +1775,7 @@ snapshots: process-warning@4.0.1: {} - process-warning@5.0.0: {} + process-warning@5.1.1: {} quick-format-unescaped@4.0.4: {} From 55732a901d8aa3944ed20697a9697fd985ddabd5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 14:25:11 +0200 Subject: [PATCH 80/85] Presence: the publisher sequence is in-process, the state root keeps no owner-only file, an Idunn generation never self-enrolls nextPublisherSequence read and rewrote /runtime-presence-sequence at mode 0600, which the next generation (a fresh DynamicUser uid in the shared state group) cannot open. Odin compares publisher_sequence only within one activation and Idunn mints a new activation per start, so PublisherSequence derives it from the observation clock, never below the previous value plus one, as Muninn does. openProviderHealthIdentity under GAMECULT_IDUNN_RUNTIME_BUNDLE now refuses when the presence descriptor is absent instead of enrolling an identity Idunn does not trust. Adds the cross-generation test (root only) and the sequence, no-dataRoot and missing-descriptor tests. Co-Authored-By: Claude Opus 5.5 --- src/index.ts | 8 +- src/provider-health-identity.ts | 8 ++ src/runtime-presence.ts | 41 ++++----- tests/fixtures/presence-descriptors.ts | 72 +++++++++++++++ tests/fixtures/state-root-generation-child.ts | 58 +++++++++++++ tests/presence-identity-descriptor.test.ts | 28 +++++- tests/publisher-sequence.test.ts | 87 +++++++++++++++++++ tests/state-root-generations.test.ts | 81 +++++++++++++++++ 8 files changed, 351 insertions(+), 32 deletions(-) create mode 100644 tests/fixtures/presence-descriptors.ts create mode 100644 tests/fixtures/state-root-generation-child.ts create mode 100644 tests/publisher-sequence.test.ts create mode 100644 tests/state-root-generations.test.ts diff --git a/src/index.ts b/src/index.ts index 4c89a70..8ee966b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ -import path from "node:path"; import { resolveWriteLease, WriteLeaseNotHeldError } from "./process-write-lease.js"; import { buildApp } from "./app.js"; import { loadConfig } from "./config.js"; @@ -9,7 +8,7 @@ import type { CultNetOperationServer } from "cultnet-ts"; import { createHeimdallRuntimePulse, buildHeimdallHealthDetail, publishHeimdallVerseState } from "./verse-state.js"; import { buildPresenceStatement, - nextPublisherSequence, + PublisherSequence, readRuntimeBundleFacts, type RuntimeBundleFacts, } from "./runtime-presence.js"; @@ -76,6 +75,7 @@ async function publishVerseState(): Promise { * work that can only ever return the same answer or fail. */ let runtimeBundleFacts: RuntimeBundleFacts | undefined; +const presenceSequence = new PublisherSequence(); async function publishHealthState(): Promise { const pulse = createHeimdallRuntimePulse(config); @@ -111,9 +111,7 @@ async function publishHealthState(): Promise { // refuses the pair, and Idunn needs the warming statement to promote it. writeLeaseSha256: null, observedAtUnixMillis, - publisherSequence: await nextPublisherSequence( - path.join(config.dataRoot, "runtime-presence-sequence"), - ), + publisherSequence: presenceSequence.next(observedAtUnixMillis), providerHealthIdentityPath: config.providerHealthIdentityPath, }); diff --git a/src/provider-health-identity.ts b/src/provider-health-identity.ts index 847d39c..ad4903f 100644 --- a/src/provider-health-identity.ts +++ b/src/provider-health-identity.ts @@ -128,6 +128,14 @@ export async function openProviderHealthIdentity( ): Promise { const descriptor = passedDescriptor(PRESENCE_IDENTITY_FD_NAME, env); if (descriptor === undefined) { + if (env.GAMECULT_IDUNN_RUNTIME_BUNDLE) { + // Inside an Idunn generation a self-enrolled key is one Idunn does not + // trust, and its private store would land in a state root the next + // generation must share. The descriptor is the only identity there. + throw new Error( + `Idunn runtime bundle is set but the ${PRESENCE_IDENTITY_FD_NAME} descriptor was not passed; refusing to self-enroll` + ); + } return openOrEnrollProviderHealthIdentity(path); } diff --git a/src/runtime-presence.ts b/src/runtime-presence.ts index 8cf0615..e3ed0b1 100644 --- a/src/runtime-presence.ts +++ b/src/runtime-presence.ts @@ -20,7 +20,6 @@ import path from "node:path"; import { createHash, createPrivateKey, type KeyObject } from "node:crypto"; import { readFileSync } from "node:fs"; -import { readFile, writeFile, rename } from "node:fs/promises"; import { decode } from "@msgpack/msgpack"; import { @@ -201,35 +200,25 @@ export function openActivationSigner(env: NodeJS.ProcessEnv = process.env): KeyO } /** - * The publisher sequence must strictly increase for the life of a signer - * identity, because Idunn treats a repeat or a regression as a replay and - * refuses the observation. + * The publisher sequence a process stamps on its presence statements. * - * An in-process counter is therefore not enough: a restart would reset it and - * every statement afterwards would be refused until the count climbed past the - * old high-water mark. It lives beside the target's state instead, and is - * advanced before use. + * Odin compares `publisher_sequence` only against stored presence of the same + * activation, and Idunn mints a fresh activation for every start, continuity + * restarts included. So the sequence need only strictly increase within this + * process; nothing has to survive a restart, and nothing is written to the + * state root. Muninn derives its sequence the same way: the observation clock, + * never below the previous value plus one. */ -export async function nextPublisherSequence(sequencePath: string): Promise { - let current = 0; - try { - const raw = (await readFile(sequencePath, "utf8")).trim(); - const parsed = Number(raw); - if (Number.isSafeInteger(parsed) && parsed > 0) { - current = parsed; - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; +export class PublisherSequence { + private last = 0; + + next(observedAtUnixMillis: number): number { + if (!Number.isSafeInteger(observedAtUnixMillis)) { + throw new Error("observedAtUnixMillis is not a safe integer"); } + this.last = Math.max(observedAtUnixMillis, this.last + 1); + return this.last; } - const next = current + 1; - // Written through a temporary file: a torn write here would look like a - // regression and cost the target every future observation. - const temporary = `${sequencePath}.next`; - await writeFile(temporary, `${next}\n`, { mode: 0o600 }); - await rename(temporary, sequencePath); - return next; } export interface PresenceStatementInputs { diff --git a/tests/fixtures/presence-descriptors.ts b/tests/fixtures/presence-descriptors.ts new file mode 100644 index 0000000..0448330 --- /dev/null +++ b/tests/fixtures/presence-descriptors.ts @@ -0,0 +1,72 @@ +/** + * What Idunn hands a generation for runtime presence: a parent-only identity + * store and an activation seed, passed as descriptors named in LISTEN_FDNAMES. + * Every key here is generated for the test or a canary byte pattern. + */ +import { closeSync, openSync } from "node:fs"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { encode } from "@msgpack/msgpack"; +import { SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; +import { IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME } from "cultnet-ts"; + +import { openOrEnrollProviderHealthIdentity } from "../../src/provider-health-identity.js"; +import type { RuntimeBundleFacts } from "../../src/runtime-presence.js"; + +export const bundleFacts: RuntimeBundleFacts = { + target: "heimdall", + planId: "plan-1", + incarnationId: "incarnation-1", + sealedReleaseId: "release-1", + runtimeId: "runtime-1", + runtimeInstanceId: "instance-1", + expectedProjectionSha256: "sha256-projection", + activationWitnessSha256: "sha256-witness", + activationSignerIdentityId: "activation-signer", + stateSchemaGeneration: null, + stateContractSha256: null, + capabilities: [], + healthContract: "health-contract", +}; + +/** Write both stores into `dir`, readable by any uid (the children run as others). */ +export async function writePresenceFixtures(dir: string): Promise { + await mkdir(dir, { recursive: true }); + await chmod(dir, 0o755); + const enrolled = path.join(dir, "enrolled.cc"); + await openOrEnrollProviderHealthIdentity(enrolled); + const [record] = await new SingleFileMessagePackBackingStore(enrolled).pullAll(); + // Re-wrapped the way cultcache-rs writes a store, which is how Idunn writes it. + await writeFile( + path.join(dir, "runtime-presence-identity.cc"), + encode([ + [ + "gamecult-provider-health-identity", + "gamecult.provider_health_identity.private.v1", + record!.payload, + "2026-09-06T00:00:00.000Z", + "gamecult.provider_health_identity.private.v1", + ], + ]), + { mode: 0o644 } + ); + await writeFile(path.join(dir, "activation-seed"), Buffer.alloc(32, 7), { mode: 0o644 }); +} + +/** Open both descriptors and name them the way systemd's LISTEN_FDNAMES does. */ +export function openPresenceDescriptors(dir: string): { env: NodeJS.ProcessEnv; close: () => void } { + const fds = [ + [openSync(path.join(dir, "runtime-presence-identity.cc"), "r"), "gamecult-runtime-presence-identity"], + [openSync(path.join(dir, "activation-seed"), "r"), IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME], + ] as const; + const names: string[] = []; + for (const [fd, name] of fds) { + while (names.length < fd - 3) names.push(`pad-${names.length}`); + names[fd - 3] = name; + } + return { + env: { LISTEN_FDNAMES: names.join(":") }, + close: () => fds.forEach(([fd]) => closeSync(fd)), + }; +} diff --git a/tests/fixtures/state-root-generation-child.ts b/tests/fixtures/state-root-generation-child.ts new file mode 100644 index 0000000..0e53ade --- /dev/null +++ b/tests/fixtures/state-root-generation-child.ts @@ -0,0 +1,58 @@ +/** + * One Idunn generation, as a process: it runs as the uid and gid its parent + * gave it, under UMask=0007, and does what Heimdall does to a state root on a + * pulse. The parent test spawns two of these under different uids that share + * a group. + * + * argv: + */ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { loadConfig } from "../../src/config.js"; +import { PublisherSequence, buildPresenceStatement } from "../../src/runtime-presence.js"; +import { bundleFacts, openPresenceDescriptors } from "./presence-descriptors.js"; +import { createHeimdallRuntimePulse, publishHeimdallVerseState } from "../../src/verse-state.js"; + +const [role, stateRoot, fixtureDir] = process.argv.slice(2) as [string, string, string]; +process.umask(0o007); + +const config = loadConfig({ + GC_ACCESS_BASE_URL: "https://heimdall.example.test", + GC_ACCESS_DATA_ROOT: stateRoot, +}); + +async function publishPresence(): Promise { + const descriptors = openPresenceDescriptors(fixtureDir); + try { + const observedAtUnixMillis = Date.now(); + await buildPresenceStatement({ + bundle: bundleFacts, + state: "warming", + detail: "generation probe", + boundEndpoint: null, + writeLeaseSha256: null, + observedAtUnixMillis, + publisherSequence: new PublisherSequence().next(observedAtUnixMillis), + providerHealthIdentityPath: path.join(stateRoot, "provider-health-identity.cc"), + env: descriptors.env, + }); + } finally { + descriptors.close(); + } +} + +if (role === "next") { + // Generation N+1 reads what N left before it replaces it. + const previous = await readFile(config.cultCachePath); + if (previous.length === 0) throw new Error("previous generation's snapshot is empty"); +} + +await publishHeimdallVerseState(config, createHeimdallRuntimePulse(config)); +await publishPresence(); + +if (role === "first") { + // The 0600 counter the 2026-09-07 generation left behind. Nothing reads it + // now; it stays as the stale fixture the next generation must live beside. + await writeFile(path.join(stateRoot, "runtime-presence-sequence"), "41\n", { mode: 0o600 }); +} diff --git a/tests/presence-identity-descriptor.test.ts b/tests/presence-identity-descriptor.test.ts index 97c5233..731d16e 100644 --- a/tests/presence-identity-descriptor.test.ts +++ b/tests/presence-identity-descriptor.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { access, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { openSync, closeSync } from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -112,3 +112,29 @@ describe("presence identity descriptor selection", () => { }); }); }); + +describe("an Idunn generation without the presence identity descriptor", () => { + it("refuses, naming the descriptor, and enrolls nothing", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-nodescriptor-")); + const identityPath = path.join(dir, "provider-health-identity.cc"); + + await expect( + openProviderHealthIdentity(identityPath, { + GAMECULT_IDUNN_RUNTIME_BUNDLE: "/run/idunn/bundle", + LISTEN_FDNAMES: "some-other-descriptor", + }) + ).rejects.toThrow("gamecult-runtime-presence-identity"); + + await expect(access(identityPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("still self-enrolls outside Idunn", async () => { + if (!machineIdAvailable) return; + const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-outside-idunn-")); + const identityPath = path.join(dir, "provider-health-identity.cc"); + + await openProviderHealthIdentity(identityPath, {}); + + await expect(access(identityPath)).resolves.toBeUndefined(); + }); +}); diff --git a/tests/publisher-sequence.test.ts b/tests/publisher-sequence.test.ts new file mode 100644 index 0000000..9a4f147 --- /dev/null +++ b/tests/publisher-sequence.test.ts @@ -0,0 +1,87 @@ +import { mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { loadConfig } from "../src/config.js"; +import { PublisherSequence, buildPresenceStatement } from "../src/runtime-presence.js"; +import { bundleFacts, openPresenceDescriptors, writePresenceFixtures } from "./fixtures/presence-descriptors.js"; + +const machineIdAvailable = await readFile("/etc/machine-id", "utf8").then( + () => true, + () => false +); + +describe("PublisherSequence", () => { + it("starts at the observed millis", () => { + expect(new PublisherSequence().next(1_700_000_000_000)).toBe(1_700_000_000_000); + }); + + it("strictly increases for equal, earlier and later observations", () => { + const sequence = new PublisherSequence(); + const first = sequence.next(1_700_000_000_000); + const equal = sequence.next(1_700_000_000_000); + const earlier = sequence.next(1_699_999_990_000); + const later = sequence.next(1_700_000_050_000); + expect([first, equal, earlier, later]).toEqual([ + 1_700_000_000_000, + 1_700_000_000_001, + 1_700_000_000_002, + 1_700_000_050_000, + ]); + }); + + it("never falls below the observed millis once the clock moves ahead", () => { + const sequence = new PublisherSequence(); + sequence.next(1_700_000_000_000); + sequence.next(1_700_000_000_000); + sequence.next(1_700_000_000_000); + expect(sequence.next(1_700_000_000_002)).toBe(1_700_000_000_003); + expect(sequence.next(1_700_000_000_010)).toBe(1_700_000_000_010); + }); + + it("refuses a value that is not a safe integer and keeps its place", () => { + const sequence = new PublisherSequence(); + expect(sequence.next(10)).toBe(10); + for (const bad of [Number.NaN, Number.POSITIVE_INFINITY, 1.5, 2 ** 60]) { + expect(() => sequence.next(bad)).toThrow("observedAtUnixMillis"); + } + expect(sequence.next(5)).toBe(11); + }); +}); + +describe.skipIf(!machineIdAvailable)("presence publication and the state root", () => { + it("touches no path under dataRoot", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-nodataroot-")); + await writePresenceFixtures(path.join(dir, "fixtures")); + // A regular file as dataRoot: opening anything beneath it fails with + // ENOTDIR for every uid, root included. + const dataRoot = path.join(dir, "not-a-directory"); + await writeFile(dataRoot, ""); + const config = loadConfig({ + GC_ACCESS_BASE_URL: "https://heimdall.example.test", + GC_ACCESS_DATA_ROOT: dataRoot, + }); + + const descriptors = openPresenceDescriptors(path.join(dir, "fixtures")); + try { + const observedAtUnixMillis = Date.parse("2026-10-23T12:00:00.000Z"); + const statement = await buildPresenceStatement({ + bundle: bundleFacts, + state: "warming", + detail: "probe", + boundEndpoint: null, + writeLeaseSha256: null, + observedAtUnixMillis, + publisherSequence: new PublisherSequence().next(observedAtUnixMillis), + providerHealthIdentityPath: config.providerHealthIdentityPath, + env: descriptors.env, + }); + expect(statement.target).toBe("heimdall"); + } finally { + descriptors.close(); + } + expect((await readdir(dir)).sort()).toEqual(["fixtures", "not-a-directory"]); + }); +}); diff --git a/tests/state-root-generations.test.ts b/tests/state-root-generations.test.ts new file mode 100644 index 0000000..7632f14 --- /dev/null +++ b/tests/state-root-generations.test.ts @@ -0,0 +1,81 @@ +import { execFile } from "node:child_process"; +import { chmod, chown, mkdir, mkdtemp, readdir, stat } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { writePresenceFixtures } from "./fixtures/presence-descriptors.js"; + +const run = promisify(execFile); + +const STATE_GID = 4242; +const GENERATION_N = 61184; +const GENERATION_N_PLUS_1 = 61185; + +// Creating uids and groups needs root; the stopgap's node container runs as root. +const isRoot = process.getuid?.() === 0; + +async function walk(dir: string): Promise { + const found: string[] = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + found.push(full); + if (entry.isDirectory()) found.push(...(await walk(full))); + } + return found; +} + +/** + * Idunn gives every generation a fresh DynamicUser uid and one shared group, + * UMask=0007, and a 2770 setgid state root. Generation N+1 runs beside or after + * N and must be able to read and replace everything N left there. + */ +describe.skipIf(!isRoot)("Idunn state root across generations", () => { + it("generation N+1, a different uid in the same group, reads and replaces generation N's state", async () => { + const base = await mkdtemp(path.join(os.tmpdir(), "heimdall-generations-")); + await chmod(base, 0o755); + + // The fixtures stand in for what Idunn passes as descriptors. + const fixtures = path.join(base, "fixtures"); + await writePresenceFixtures(fixtures); + + const stateRoot = path.join(base, "state"); + await mkdir(stateRoot); + await chown(stateRoot, 0, STATE_GID); + await chmod(stateRoot, 0o2770); + + const child = path.join(import.meta.dirname, "fixtures", "state-root-generation-child.ts"); + const generation = (role: "first" | "next", uid: number) => + run(process.execPath, ["--import", "tsx", child, role, stateRoot, fixtures], { + uid, + gid: STATE_GID, + cwd: path.resolve(import.meta.dirname, ".."), + env: { PATH: process.env.PATH ?? "", HOME: fixtures }, + }); + + await generation("first", GENERATION_N); + + const left = await walk(stateRoot); + expect(left.map((entry) => path.relative(stateRoot, entry)).sort()).toEqual([ + "cultcache", + path.join("cultcache", "heimdall.service.cc"), + "runtime-presence-sequence", + ]); + + await generation("next", GENERATION_N_PLUS_1); + + // Generation N+1 replaced the snapshot, and no owner-only file is left + // behind for generation N+2: everything but the stale 0600 fixture is + // readable and writable by the state group. + const snapshot = await stat(path.join(stateRoot, "cultcache", "heimdall.service.cc")); + expect(snapshot.uid).toBe(GENERATION_N_PLUS_1); + expect(snapshot.gid).toBe(STATE_GID); + for (const entry of await walk(stateRoot)) { + if (path.basename(entry) === "runtime-presence-sequence") continue; + expect((await stat(entry)).mode & 0o060, path.relative(stateRoot, entry)).toBe(0o060); + } + expect((await readdir(stateRoot)).sort()).toEqual(["cultcache", "runtime-presence-sequence"]); + }, 60_000); +}); From e8548614c8222658d46ad5e0e7bd57e34dd34cd6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 14:26:13 +0200 Subject: [PATCH 81/85] Generation test: bundle the generation process so a uid that cannot read the checkout can run it Co-Authored-By: Claude Opus 5.5 --- tests/state-root-generations.test.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/state-root-generations.test.ts b/tests/state-root-generations.test.ts index 7632f14..b8424f3 100644 --- a/tests/state-root-generations.test.ts +++ b/tests/state-root-generations.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; +import { build } from "esbuild"; import { describe, expect, it } from "vitest"; import { writePresenceFixtures } from "./fixtures/presence-descriptors.js"; @@ -46,12 +47,28 @@ describe.skipIf(!isRoot)("Idunn state root across generations", () => { await chown(stateRoot, 0, STATE_GID); await chmod(stateRoot, 0o2770); - const child = path.join(import.meta.dirname, "fixtures", "state-root-generation-child.ts"); + // The generations run as uids that cannot read the checkout, so each runs + // one self-contained bundle built into the world-readable scratch root. + const child = path.join(base, "generation.mjs"); + await build({ + entryPoints: [path.join(import.meta.dirname, "fixtures", "state-root-generation-child.ts")], + bundle: true, + platform: "node", + format: "esm", + target: "node24", + outfile: child, + external: ["pg-native"], + banner: { + js: "import{createRequire}from'node:module';const require=createRequire(import.meta.url);", + }, + logLevel: "silent", + }); + await chmod(child, 0o644); const generation = (role: "first" | "next", uid: number) => - run(process.execPath, ["--import", "tsx", child, role, stateRoot, fixtures], { + run(process.execPath, [child, role, stateRoot, fixtures], { uid, gid: STATE_GID, - cwd: path.resolve(import.meta.dirname, ".."), + cwd: base, env: { PATH: process.env.PATH ?? "", HOME: fixtures }, }); From c0d30e80816053cd151f0789e7f4bcdf5c6c5ac1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 17:48:02 +0200 Subject: [PATCH 82/85] Test fixtures: the Rust owner's Idunn runtime records, and one fake Idunn RUDP endpoint tests/fixtures/idunn-runtime holds expected.cc, activation.cc, the activation seed, the presence identity credential and the machine-id it is bound to, copied from CultLib 217a9480 (Rust-written; test keys only). fake-idunn-rudp.ts is the accept server idunn-rudp-health.test.ts built inline, now keeping a session per remote and every delivered frame, so a boot test can receive several pulses. Co-Authored-By: Claude Opus 5.5 --- tests/fixtures/fake-idunn-rudp.ts | 64 ++++++++++++++++++ tests/fixtures/idunn-runtime/.gitattributes | 1 + tests/fixtures/idunn-runtime/README.md | 11 +++ .../idunn-runtime/activation-key.seed | 1 + tests/fixtures/idunn-runtime/activation.cc | Bin 0 -> 798 bytes tests/fixtures/idunn-runtime/expected.cc | Bin 0 -> 1067 bytes tests/fixtures/idunn-runtime/machine-id | 1 + .../provider-identity.credential | Bin 0 -> 682 bytes tests/idunn-rudp-health.test.ts | 41 ++--------- 9 files changed, 83 insertions(+), 36 deletions(-) create mode 100644 tests/fixtures/fake-idunn-rudp.ts create mode 100644 tests/fixtures/idunn-runtime/.gitattributes create mode 100644 tests/fixtures/idunn-runtime/README.md create mode 100644 tests/fixtures/idunn-runtime/activation-key.seed create mode 100644 tests/fixtures/idunn-runtime/activation.cc create mode 100644 tests/fixtures/idunn-runtime/expected.cc create mode 100644 tests/fixtures/idunn-runtime/machine-id create mode 100644 tests/fixtures/idunn-runtime/provider-identity.credential diff --git a/tests/fixtures/fake-idunn-rudp.ts b/tests/fixtures/fake-idunn-rudp.ts new file mode 100644 index 0000000..e084666 --- /dev/null +++ b/tests/fixtures/fake-idunn-rudp.ts @@ -0,0 +1,64 @@ +/** + * A stand-in for Idunn's health endpoint: it accepts each RUDP connect, acks + * the frame that follows and keeps every delivered frame. Heimdall opens a + * fresh socket and session per publication, so sessions are kept per remote. + */ +import dgram from "node:dgram"; +import { + CultNetRudpSession, + decodeRudpPacket, + encodeRudpPacket, + type CultNetTransportFrame, +} from "cultnet-ts"; + +export interface FakeIdunnRudp { + port: number; + frames: CultNetTransportFrame[]; + /** Resolves with the nth delivered frame (0-based) once it has arrived. */ + frame(index: number): Promise; + close(): void; +} + +export async function startFakeIdunnRudp(): Promise { + const server = dgram.createSocket("udp4"); + const sessions = new Map(); + const frames: CultNetTransportFrame[] = []; + const waiting: Array<{ index: number; resolve: (frame: CultNetTransportFrame) => void }> = []; + + server.on("message", (wire, remote) => { + const packet = decodeRudpPacket(wire); + const key = `${remote.address}:${remote.port}`; + if (packet.packetType === "connect") { + const session = new CultNetRudpSession({ + connectionId: 0x1d0d0001, + initialSequence: 1, + resendDelayMs: 100, + }); + sessions.set(key, session); + server.send(encodeRudpPacket(session.acceptConnect(packet, Date.now())), remote.port, remote.address); + return; + } + const session = sessions.get(key); + if (!session) return; + const result = session.receive(packet, Date.now()); + if (result.reply) server.send(encodeRudpPacket(result.reply), remote.port, remote.address); + for (const frame of result.delivered) { + frames.push(frame); + for (const waiter of waiting.filter((entry) => entry.index === frames.length - 1)) waiter.resolve(frame); + } + if (result.delivered.length > 0) { + server.send(encodeRudpPacket(session.createAck()), remote.port, remote.address); + } + }); + await new Promise((resolve) => server.bind(0, "127.0.0.1", resolve)); + + return { + port: server.address().port, + frames, + frame: (index) => + frames[index] + ? Promise.resolve(frames[index]!) + : new Promise((resolve) => waiting.push({ index, resolve })), + close: () => server.close(), + }; +} diff --git a/tests/fixtures/idunn-runtime/.gitattributes b/tests/fixtures/idunn-runtime/.gitattributes new file mode 100644 index 0000000..fa1385d --- /dev/null +++ b/tests/fixtures/idunn-runtime/.gitattributes @@ -0,0 +1 @@ +* -text diff --git a/tests/fixtures/idunn-runtime/README.md b/tests/fixtures/idunn-runtime/README.md new file mode 100644 index 0000000..0f26117 --- /dev/null +++ b/tests/fixtures/idunn-runtime/README.md @@ -0,0 +1,11 @@ +# Idunn runtime fixtures + +Written by the Rust owner, not by Heimdall's tests. Copied byte for byte from CultLib +`217a9480`, `packages/cultnet-ts/test/fixtures/idunn-runtime/web` (`expected.cc`, +`activation.cc`, `activation-key.seed`, `provider-identity.credential`) and that directory's +`machine-id`. CultLib regenerates them with `packages/cultnet-rs/examples/idunn_runtime_fixture.rs` +(its own README has the Yggdrasil command); the provider identity is bound to `machine-id`, so a +test that decodes it supplies that machine id instead of `/etc/machine-id`. + +Every key here is a test key. `provider-identity.credential` is the shape Idunn passes on the +presence-identity descriptor: key fields are 32-integer arrays, which only the Rust owner writes. diff --git a/tests/fixtures/idunn-runtime/activation-key.seed b/tests/fixtures/idunn-runtime/activation-key.seed new file mode 100644 index 0000000..7b6d37f --- /dev/null +++ b/tests/fixtures/idunn-runtime/activation-key.seed @@ -0,0 +1 @@ +Odfl*oFvnhLpSp \ No newline at end of file diff --git a/tests/fixtures/idunn-runtime/activation.cc b/tests/fixtures/idunn-runtime/activation.cc new file mode 100644 index 0000000000000000000000000000000000000000..3931baf2d0ad2e084e2704913f368f46605cef11 GIT binary patch literal 798 zcmbtOOKTHR6mDrDh#<4Z?5WQE!R>{*&xdssSR4V zbL&ZLn)P~+HtTtu*fUb+aZBcLqaL)v?t0wQ^$2hBsYRLnv+Z)`CnR!ACbnEH7etkJ z{gEECY33&tt9`LIBsNiAU9O%ub+J=4Td)Hq2n(HUn~?E`v zrQVT=SzkshLmV-MA{Htw7-GnUMq0{&V5>0p?Z#@L)F{jeK!gRx%&~|=r1d>(!pOgR zqot9>_QEImdFkD^;OCBX=k(p>FZ1u=;rQC-5jfBo-g0N`M;GQE*vnrpwGSnj9X|SO R8N2`h literal 0 HcmV?d00001 diff --git a/tests/fixtures/idunn-runtime/expected.cc b/tests/fixtures/idunn-runtime/expected.cc new file mode 100644 index 0000000000000000000000000000000000000000..dbf7153b5b59ff258a410787f85a7f7695945346 GIT binary patch literal 1067 zcmbtTO-jQ+6jqPm2_!2QW->EP(n(f=3wI(YE)e1)f)^cUGO+_m5dg?Mh(=t^{#J6|^nyb~)X=~rkdyrdUz(`nQ=(fk)l&1Ui|13|@W26UEflF;TLMu{#!oG2Z`Jdiz{ht`DBdRc>Yh zr{(LmA{A5=mrisugai*JbRCN4Py6VY$lxiAhn7Iq_U= zUO#QO{gh{6r(E}Z-{wuh7`GkMvQ5GWYjKCumN10?>HsAz$Ft0a*YpUpOv=2bLwztC zHfAmBtxAs4JeFYudH}p5gGm?WoEQ$@`-5sSJxy{6H-VJhhtcI42BA%a)gdlp4Lq3A Yl99UT-&d&+?}m@RDb=sJ?vcfdFMc*;F8}}l literal 0 HcmV?d00001 diff --git a/tests/fixtures/idunn-runtime/machine-id b/tests/fixtures/idunn-runtime/machine-id new file mode 100644 index 0000000..a71716f --- /dev/null +++ b/tests/fixtures/idunn-runtime/machine-id @@ -0,0 +1 @@ +c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1 \ No newline at end of file diff --git a/tests/fixtures/idunn-runtime/provider-identity.credential b/tests/fixtures/idunn-runtime/provider-identity.credential new file mode 100644 index 0000000000000000000000000000000000000000..2e617139cb114b951bfdc3fca179a02ff872830c GIT binary patch literal 682 zcmb7?KWG#|6vq7nT3OhrSZQLRm_2vqW@cxuvKA}?Drv%IW_BjGve`{?dv`IF61B03 zU_fkyNRSk%0zoTn1kn#{0^_E%dsB)BeWjJe_#1P3Dl_pU%bF3|5>G`JvXU1?BAFz1=zhLBGR;Ls;j^PfzHDg6y_Lowh7cQNvvhZlz%}W;`q$V%3yqs-@BU1l_84XD1uQ33Q zdRK$@uP zrFq$_by4U#ugbknP1P^6)D~6gvex7w6(N@6B1 z#^E0aO4Im_m8;mrw!!0xKDG{HZD9%T@f}~$3~?_&+`%S3;Q0+_je?J E0GWjm%m4rY literal 0 HcmV?d00001 diff --git a/tests/idunn-rudp-health.test.ts b/tests/idunn-rudp-health.test.ts index 66127d4..fa40393 100644 --- a/tests/idunn-rudp-health.test.ts +++ b/tests/idunn-rudp-health.test.ts @@ -1,18 +1,12 @@ -import dgram from "node:dgram"; import { mkdtemp } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { decode } from "@msgpack/msgpack"; -import { - CultNetRudpSession, - decodeRudpPacket, - encodeRudpPacket, - parseCultNetMessage, - type CultNetTransportFrame, -} from "cultnet-ts"; +import { parseCultNetMessage } from "cultnet-ts"; import { describe, expect, it, vi } from "vitest"; import { loadConfig } from "../src/config.js"; import { publishIdunnRudpHealth } from "../src/idunn-rudp-health.js"; +import { startFakeIdunnRudp } from "./fixtures/fake-idunn-rudp.js"; vi.mock("node:fs/promises", async (importOriginal) => { const original = await importOriginal(); @@ -27,37 +21,12 @@ vi.mock("node:fs/promises", async (importOriginal) => { describe("Heimdall Idunn health publication", () => { it("completes the canonical RUDP handshake before publishing its signed frame", async () => { - const server = dgram.createSocket("udp4"); - const session = new CultNetRudpSession({ - connectionId: 0x1d0d0001, - initialSequence: 1, - resendDelayMs: 100, - }); - let resolveFrame!: (frame: CultNetTransportFrame) => void; - const frameReceived = new Promise((resolve) => { - resolveFrame = resolve; - }); - - server.on("message", (wire, remote) => { - const packet = decodeRudpPacket(wire); - if (packet.packetType === "connect") { - server.send(encodeRudpPacket(session.acceptConnect(packet, Date.now())), remote.port, remote.address); - return; - } - const result = session.receive(packet, Date.now()); - if (result.reply) server.send(encodeRudpPacket(result.reply), remote.port, remote.address); - for (const frame of result.delivered) resolveFrame(frame); - if (result.delivered.length > 0) { - server.send(encodeRudpPacket(session.createAck()), remote.port, remote.address); - } - }); - await new Promise((resolve) => server.bind(0, "127.0.0.1", resolve)); + const server = await startFakeIdunnRudp(); try { const tmp = await mkdtemp(path.join(os.tmpdir(), "heimdall-idunn-rudp-")); - const address = server.address(); const config = loadConfig({ - GC_ACCESS_IDUNN_RUDP_HEALTH: `127.0.0.1:${address.port}`, + GC_ACCESS_IDUNN_RUDP_HEALTH: `127.0.0.1:${server.port}`, GC_ACCESS_PROVIDER_HEALTH_IDENTITY_PATH: path.join(tmp, "provider-health-identity.cc"), }); const observedAt = "2026-08-22T00:45:00.000Z"; @@ -78,7 +47,7 @@ describe("Heimdall Idunn health publication", () => { observedAt, presence, }); - const frame = await frameReceived; + const frame = await server.frame(0); const message = parseCultNetMessage(decode(frame.payload), "cultnet.schema.v0"); expect(frame.channelId).toBe("schema"); From c0441558553e224f8fdd80d2c1124286a4a155b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 17:48:08 +0200 Subject: [PATCH 83/85] Heimdall opens its Idunn presence authority once, before it listens index.ts declared the presence sequence below the startup try, so the bundle's hoisted var was undefined at the first publish; decodePrivateIdentity refused the Rust owner's integer-array key fields; both descriptors were re-read every pulse and returned nothing after the first. openPresenceAuthority now reads the bundle facts and each descriptor once (whole, from offset 0, then closed), owns the one publisher sequence, and signs through them; a launch-contract fault throws before anything listens, so the process exits non-zero. decodePrivateIdentity takes the shape of its source and names the failed check, never a value. openActivationSigner and the per-pulse descriptor reads are deleted. tests/idunn-boot.test.ts boots the bundle under the Idunn launch contract and checks the signed warming frames and the fail-fast exits. The descriptor, sequence and generation tests now use the Rust-written fixtures. Co-Authored-By: Claude Opus 5.5 --- src/index.ts | 43 +-- src/provider-health-identity.ts | 96 ++++-- src/runtime-presence.ts | 118 ++++--- tests/fixtures/presence-descriptors.ts | 79 ++--- tests/fixtures/state-root-generation-child.ts | 27 +- tests/idunn-boot.test.ts | 302 ++++++++++++++++++ tests/presence-identity-descriptor.test.ts | 121 ++++--- tests/publisher-sequence.test.ts | 44 ++- 8 files changed, 606 insertions(+), 224 deletions(-) create mode 100644 tests/idunn-boot.test.ts diff --git a/src/index.ts b/src/index.ts index 8ee966b..0bc13c3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,14 +6,16 @@ import { publishHeimdallOdinState } from "./odin-publication.js"; import { startHeimdallPrivateCommandPlane } from "./private-command-plane.js"; import type { CultNetOperationServer } from "cultnet-ts"; import { createHeimdallRuntimePulse, buildHeimdallHealthDetail, publishHeimdallVerseState } from "./verse-state.js"; -import { - buildPresenceStatement, - PublisherSequence, - readRuntimeBundleFacts, - type RuntimeBundleFacts, -} from "./runtime-presence.js"; +import { openPresenceAuthority } from "./runtime-presence.js"; const config = loadConfig(); +// Under Idunn the launch's presence keys, bundle facts and publisher sequence +// open here, before anything listens. A launch-contract fault throws, uncaught, +// and the process exits non-zero with no listener: Idunn fails Warming in +// seconds rather than waiting on a candidate that can never sign. +const presenceAuthority = config.idunnRuntimeBundlePath + ? await openPresenceAuthority(config) + : undefined; const app = await buildApp({ config }); const versePulseIntervalMs = 60_000; const healthPulseIntervalMs = 10_000; @@ -69,14 +71,6 @@ async function publishVerseState(): Promise { await publishHeimdallOdinState(config, pulse); } -/** - * Read once and keep: the bundle is Idunn's immutable record of this launch and - * cannot change while the process lives, so re-reading it every pulse would be - * work that can only ever return the same answer or fail. - */ -let runtimeBundleFacts: RuntimeBundleFacts | undefined; -const presenceSequence = new PublisherSequence(); - async function publishHealthState(): Promise { const pulse = createHeimdallRuntimePulse(config); @@ -94,30 +88,17 @@ async function publishHealthState(): Promise { ? buildHeimdallHealthDetail(config, pulse) : `Heimdall candidate warming; ${lease.reason}`; - if (!config.idunnRuntimeBundlePath) { + if (!presenceAuthority) { // Outside Idunn there is no launch to attest and nothing that admits the // statement; publishing is simply not part of that world. return; } - runtimeBundleFacts ??= await readRuntimeBundleFacts(config.idunnRuntimeBundlePath); - - const observedAtUnixMillis = Date.parse(pulse.updatedAt); - const presence = await buildPresenceStatement({ - bundle: runtimeBundleFacts, - state: lease.mayWrite ? "active" : "warming", - detail, - boundEndpoint: `http://${config.host}:${config.port}`, - // A warming candidate holds no lease and must not claim one; the contract - // refuses the pair, and Idunn needs the warming statement to promote it. - writeLeaseSha256: null, - observedAtUnixMillis, - publisherSequence: presenceSequence.next(observedAtUnixMillis), - providerHealthIdentityPath: config.providerHealthIdentityPath, - }); + const state = lease.mayWrite ? "active" : "warming"; + const presence = presenceAuthority.sign(state, detail, Date.parse(pulse.updatedAt)); await publishIdunnRudpHealth(config, { daemonId: config.daemonId, - state: lease.mayWrite ? "active" : "warming", + state, detail, observedAt: pulse.updatedAt, presence, diff --git a/src/provider-health-identity.ts b/src/provider-health-identity.ts index ad4903f..ca89f2c 100644 --- a/src/provider-health-identity.ts +++ b/src/provider-health-identity.ts @@ -7,7 +7,7 @@ import { sign, type KeyObject, } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { closeSync, fstatSync, readSync } from "node:fs"; import { chmod, readFile } from "node:fs/promises"; import { SingleFileMessagePackBackingStore, type CultCacheEnvelope } from "@gamecult/cultcache-ts"; import { readPrivateStoreRecord } from "./idunn-store.js"; @@ -71,7 +71,7 @@ export async function openOrEnrollProviderHealthIdentity(path: string): Promise< if (existing.length !== 1) { throw new Error("Provider-health identity store must contain exactly one record."); } - return decodePrivateIdentity(existing[0]!.payload, await readMachineId()); + return decodePrivateIdentity(existing[0]!.payload, await readMachineId(), "bin"); } const PRESENCE_IDENTITY_FD_NAME = "gamecult-runtime-presence-identity"; @@ -124,7 +124,8 @@ export function passedDescriptor( */ export async function openProviderHealthIdentity( path: string, - env: NodeJS.ProcessEnv = process.env + env: NodeJS.ProcessEnv = process.env, + machineId: () => Promise = readMachineId ): Promise { const descriptor = passedDescriptor(PRESENCE_IDENTITY_FD_NAME, env); if (descriptor === undefined) { @@ -140,11 +141,33 @@ export async function openProviderHealthIdentity( } const payload = readPrivateStoreRecord( - readFileSync(descriptor), + readPassedDescriptor(descriptor), "Idunn runtime presence identity" ); - return decodePrivateIdentity(payload, await readMachineId()); + // The descriptor's writer is the Rust owner, whose key fields are integer arrays. + return decodePrivateIdentity(payload, await machineId(), "int-array"); +} + +/** + * Read a passed descriptor whole, from offset 0, and close it. A descriptor is + * a one-shot input: reading it again starts at the offset the last read left, + * which is the end, so the owner reads once at open and keeps what it decoded. + */ +export function readPassedDescriptor(descriptor: number): Buffer { + try { + const size = fstatSync(descriptor).size; + const bytes = Buffer.alloc(size); + let read = 0; + while (read < size) { + const count = readSync(descriptor, bytes, read, size - read, read); + if (count === 0) break; + read += count; + } + return bytes.subarray(0, read); + } finally { + closeSync(descriptor); + } } export function signProviderHealthPayload(identity: PrivateIdentity, payload: Uint8Array): Uint8Array { @@ -176,23 +199,58 @@ export function providerHealthPublicIdentity(identity: PrivateIdentity): { return { identityId: identity.identityId, publicKeyHex: Buffer.from(identity.publicKey).toString("hex") }; } -function decodePrivateIdentity(payload: Uint8Array, machineId: string): PrivateIdentity { +/** + * Which writer produced the record. The self-enrolling store this process + * writes keeps key fields as MessagePack bin; the Rust owner (cultnet-rs + * service_identity.rs) serialises a Vec without serde_bytes, so its key + * fields are arrays of integers. Each source is read in its own shape and the + * other is refused. + */ +type KeyShape = "bin" | "int-array"; + +function keyBytes(value: unknown, shape: KeyShape, field: string): Uint8Array { + const bytes = + shape === "bin" + ? value instanceof Uint8Array + ? value + : undefined + : Array.isArray(value) && + value.every((byte) => typeof byte === "number" && Number.isInteger(byte) && byte >= 0 && byte <= 255) + ? Uint8Array.from(value as number[]) + : undefined; + if (bytes === undefined || bytes.length !== 32) { + throw new Error( + `Provider-health identity key shape check failed: ${field} is not a 32-byte ${shape === "bin" ? "binary" : "integer-array"} key field.` + ); + } + return bytes; +} + +function decodePrivateIdentity(payload: Uint8Array, machineId: string, shape: KeyShape): PrivateIdentity { const value = decode(payload); if (!Array.isArray(value) || value.length !== 10) { - throw new Error("Provider-health identity payload is malformed."); + throw new Error("Provider-health identity schema check failed: payload is not a 10-field record."); } - const [schema, identityId, publicKey, protectedSeed, protector, binding, version] = value; - if ( - schema !== PRIVATE_SCHEMA || - typeof identityId !== "string" || - !(publicKey instanceof Uint8Array) || publicKey.length !== 32 || - !(protectedSeed instanceof Uint8Array) || protectedSeed.length !== 32 || - protector !== "linux_file_mode_machine_id_binding" || - binding !== protectorBinding(machineId) || - version !== "v1" || - identityId !== deriveIdentityId(publicKey) - ) { - throw new Error("Provider-health identity does not belong to this host or profile."); + const [schema, identityId, publicKeyField, protectedSeedField, protector, binding, version] = value; + if (schema !== PRIVATE_SCHEMA) { + throw new Error("Provider-health identity schema check failed: unexpected schema id."); + } + if (typeof identityId !== "string") { + throw new Error("Provider-health identity id check failed: id is not a string."); + } + const publicKey = keyBytes(publicKeyField, shape, "public key"); + const protectedSeed = keyBytes(protectedSeedField, shape, "protected seed"); + if (protector !== "linux_file_mode_machine_id_binding") { + throw new Error("Provider-health identity protector check failed: unexpected protector."); + } + if (binding !== protectorBinding(machineId)) { + throw new Error("Provider-health identity binding check failed: not bound to this host's machine id."); + } + if (version !== "v1") { + throw new Error("Provider-health identity version check failed: unexpected version."); + } + if (identityId !== deriveIdentityId(publicKey)) { + throw new Error("Provider-health identity id check failed: id does not derive from the public key."); } const seed = xorSeed(protectedSeed, binding); const privateKey = privateKeyFromSeed(seed); diff --git a/src/runtime-presence.ts b/src/runtime-presence.ts index e3ed0b1..3c1054b 100644 --- a/src/runtime-presence.ts +++ b/src/runtime-presence.ts @@ -19,7 +19,6 @@ import path from "node:path"; import { createHash, createPrivateKey, type KeyObject } from "node:crypto"; -import { readFileSync } from "node:fs"; import { decode } from "@msgpack/msgpack"; import { @@ -33,11 +32,14 @@ import { type RuntimePresenceState, } from "cultnet-ts"; +import type { HeimdallConfig } from "./config.js"; import { readIdunnStoreRecord } from "./idunn-store.js"; import { openProviderHealthIdentity, passedDescriptor, + readPassedDescriptor, signWithIdentity, + type PrivateIdentity, } from "./provider-health-identity.js"; const PKCS8_ED25519_SEED_PREFIX = Buffer.from("302e020100300506032b657004220420", "hex"); @@ -176,29 +178,6 @@ export async function readRuntimeBundleFacts(bundlePath: string): Promise { - const env = inputs.env ?? process.env; - const activationKey = openActivationSigner(env); - if (activationKey === undefined) { - throw new Error( - "Idunn passed no runtime activation credential, so this process cannot prove it is the " + - "incarnation Idunn started", - ); - } - const identity = await openProviderHealthIdentity(inputs.providerHealthIdentityPath, env); +export function buildPresenceStatement(inputs: PresenceStatementInputs): PresenceStatement { + const { identity } = inputs; const presence: RuntimePresenceHealth = { target: inputs.bundle.target, @@ -284,7 +253,7 @@ export async function buildPresenceStatement( const proof = runtimePresenceProofPayload(presence); const signature = signWithIdentity(identity, runtimePresenceProviderSigningMessage(proof)); const activationSignature = signWithIdentity( - { ...identity, privateKey: activationKey }, + { ...identity, privateKey: inputs.activationKey }, runtimePresenceActivationSigningMessage(proof), ); @@ -293,3 +262,72 @@ export async function buildPresenceStatement( payload: encodeRuntimePresenceHealth(presence, signature, activationSignature), }; } + +export interface PresenceAuthority { + sign( + state: RuntimePresenceState, + detail: string, + observedAtUnixMillis: number, + ): PresenceStatement; +} + +/** + * Open everything this launch's presence statements stand on, once, before the + * process listens: Idunn's bundle facts, the activation seed and the presence + * identity (each descriptor is read whole and closed), and the one publisher + * sequence. It owns them for the life of the process; a pulse supplies only its + * state, detail and observation time. + * + * A launch-contract fault (a missing or malformed bundle, descriptor or + * identity) throws here, so the process exits before it accepts a request and + * Idunn fails Warming at once, instead of a listener that can never sign. + */ +export async function openPresenceAuthority( + config: Pick, + env: NodeJS.ProcessEnv = process.env, + machineId?: () => Promise, +): Promise { + if (!config.idunnRuntimeBundlePath) { + throw new Error("no Idunn runtime bundle path: presence has no launch to attest"); + } + const bundle = await readRuntimeBundleFacts(config.idunnRuntimeBundlePath); + + const activationDescriptor = passedDescriptor(IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME, env); + if (activationDescriptor === undefined) { + throw new Error( + "Idunn passed no runtime activation credential, so this process cannot prove it is the " + + "incarnation Idunn started", + ); + } + const seed = readPassedDescriptor(activationDescriptor); + if (seed.length !== 32) { + throw new Error("Idunn runtime activation credential is not a 32-byte seed"); + } + const activationKey = createPrivateKey({ + key: Buffer.concat([PKCS8_ED25519_SEED_PREFIX, seed]), + format: "der", + type: "pkcs8", + }); + + const identity = await openProviderHealthIdentity(config.providerHealthIdentityPath, env, machineId); + const boundEndpoint = `http://${config.host}:${config.port}`; + const sequence = new PublisherSequence(); + + return { + sign: (state, detail, observedAtUnixMillis) => + buildPresenceStatement({ + bundle, + identity, + activationKey, + state, + detail, + boundEndpoint, + // A warming candidate holds no lease and must not claim one; the + // contract refuses the pair, and Idunn needs the warming statement to + // promote it. + writeLeaseSha256: null, + observedAtUnixMillis, + publisherSequence: sequence.next(observedAtUnixMillis), + }), + }; +} diff --git a/tests/fixtures/presence-descriptors.ts b/tests/fixtures/presence-descriptors.ts index 0448330..8e724f2 100644 --- a/tests/fixtures/presence-descriptors.ts +++ b/tests/fixtures/presence-descriptors.ts @@ -1,64 +1,45 @@ /** - * What Idunn hands a generation for runtime presence: a parent-only identity - * store and an activation seed, passed as descriptors named in LISTEN_FDNAMES. - * Every key here is generated for the test or a canary byte pattern. + * What Idunn hands a generation for runtime presence: the bundle directory, a + * parent-only identity store and an activation seed, the last two passed as + * descriptors named in LISTEN_FDNAMES. The bytes are the Rust owner's, copied + * from CultLib (see idunn-runtime/README.md); every key in them is a test key. */ import { closeSync, openSync } from "node:fs"; -import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { chmod, copyFile, mkdir, readFile } from "node:fs/promises"; import path from "node:path"; -import { encode } from "@msgpack/msgpack"; -import { SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; import { IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME } from "cultnet-ts"; -import { openOrEnrollProviderHealthIdentity } from "../../src/provider-health-identity.js"; -import type { RuntimeBundleFacts } from "../../src/runtime-presence.js"; +export const idunnRuntimeFixtures = path.join(import.meta.dirname, "idunn-runtime"); -export const bundleFacts: RuntimeBundleFacts = { - target: "heimdall", - planId: "plan-1", - incarnationId: "incarnation-1", - sealedReleaseId: "release-1", - runtimeId: "runtime-1", - runtimeInstanceId: "instance-1", - expectedProjectionSha256: "sha256-projection", - activationWitnessSha256: "sha256-witness", - activationSignerIdentityId: "activation-signer", - stateSchemaGeneration: null, - stateContractSha256: null, - capabilities: [], - healthContract: "health-contract", -}; +const FIXTURE_FILES = [ + "expected.cc", + "activation.cc", + "activation-key.seed", + "provider-identity.credential", + "machine-id", +] as const; -/** Write both stores into `dir`, readable by any uid (the children run as others). */ +/** The machine id the fixture identity is bound to, in place of /etc/machine-id. */ +export const fixtureMachineId = async (dir: string = idunnRuntimeFixtures): Promise => + (await readFile(path.join(dir, "machine-id"), "utf8")).trim(); + +/** Copy the fixtures into `dir`, readable by any uid (the children run as others). */ export async function writePresenceFixtures(dir: string): Promise { await mkdir(dir, { recursive: true }); await chmod(dir, 0o755); - const enrolled = path.join(dir, "enrolled.cc"); - await openOrEnrollProviderHealthIdentity(enrolled); - const [record] = await new SingleFileMessagePackBackingStore(enrolled).pullAll(); - // Re-wrapped the way cultcache-rs writes a store, which is how Idunn writes it. - await writeFile( - path.join(dir, "runtime-presence-identity.cc"), - encode([ - [ - "gamecult-provider-health-identity", - "gamecult.provider_health_identity.private.v1", - record!.payload, - "2026-09-06T00:00:00.000Z", - "gamecult.provider_health_identity.private.v1", - ], - ]), - { mode: 0o644 } - ); - await writeFile(path.join(dir, "activation-seed"), Buffer.alloc(32, 7), { mode: 0o644 }); + for (const name of FIXTURE_FILES) { + const target = path.join(dir, name); + await copyFile(path.join(idunnRuntimeFixtures, name), target); + await chmod(target, 0o644); + } } /** Open both descriptors and name them the way systemd's LISTEN_FDNAMES does. */ export function openPresenceDescriptors(dir: string): { env: NodeJS.ProcessEnv; close: () => void } { const fds = [ - [openSync(path.join(dir, "runtime-presence-identity.cc"), "r"), "gamecult-runtime-presence-identity"], - [openSync(path.join(dir, "activation-seed"), "r"), IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME], + [openSync(path.join(dir, "provider-identity.credential"), "r"), "gamecult-runtime-presence-identity"], + [openSync(path.join(dir, "activation-key.seed"), "r"), IDUNN_RUNTIME_ACTIVATION_CREDENTIAL_NAME], ] as const; const names: string[] = []; for (const [fd, name] of fds) { @@ -67,6 +48,14 @@ export function openPresenceDescriptors(dir: string): { env: NodeJS.ProcessEnv; } return { env: { LISTEN_FDNAMES: names.join(":") }, - close: () => fds.forEach(([fd]) => closeSync(fd)), + // The authority closes each descriptor it reads; this releases only what it did not. + close: () => + fds.forEach(([fd]) => { + try { + closeSync(fd); + } catch { + // already closed by the reader + } + }), }; } diff --git a/tests/fixtures/state-root-generation-child.ts b/tests/fixtures/state-root-generation-child.ts index 0e53ade..932d513 100644 --- a/tests/fixtures/state-root-generation-child.ts +++ b/tests/fixtures/state-root-generation-child.ts @@ -10,8 +10,8 @@ import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { loadConfig } from "../../src/config.js"; -import { PublisherSequence, buildPresenceStatement } from "../../src/runtime-presence.js"; -import { bundleFacts, openPresenceDescriptors } from "./presence-descriptors.js"; +import { openPresenceAuthority } from "../../src/runtime-presence.js"; +import { fixtureMachineId, openPresenceDescriptors } from "./presence-descriptors.js"; import { createHeimdallRuntimePulse, publishHeimdallVerseState } from "../../src/verse-state.js"; const [role, stateRoot, fixtureDir] = process.argv.slice(2) as [string, string, string]; @@ -25,18 +25,17 @@ const config = loadConfig({ async function publishPresence(): Promise { const descriptors = openPresenceDescriptors(fixtureDir); try { - const observedAtUnixMillis = Date.now(); - await buildPresenceStatement({ - bundle: bundleFacts, - state: "warming", - detail: "generation probe", - boundEndpoint: null, - writeLeaseSha256: null, - observedAtUnixMillis, - publisherSequence: new PublisherSequence().next(observedAtUnixMillis), - providerHealthIdentityPath: path.join(stateRoot, "provider-health-identity.cc"), - env: descriptors.env, - }); + const authority = await openPresenceAuthority( + { + idunnRuntimeBundlePath: fixtureDir, + host: config.host, + port: config.port, + providerHealthIdentityPath: path.join(stateRoot, "provider-health-identity.cc"), + }, + descriptors.env, + () => fixtureMachineId(fixtureDir), + ); + authority.sign("warming", "generation probe", Date.now()); } finally { descriptors.close(); } diff --git a/tests/idunn-boot.test.ts b/tests/idunn-boot.test.ts new file mode 100644 index 0000000..d758910 --- /dev/null +++ b/tests/idunn-boot.test.ts @@ -0,0 +1,302 @@ +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { createPublicKey, createPrivateKey, verify } from "node:crypto"; +import dgram from "node:dgram"; +import { chmod, copyFile, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { closeSync, openSync } from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { decode, encode } from "@msgpack/msgpack"; +import { + RUNTIME_PRESENCE_SLOT, + parseCultNetMessage, + runtimePresenceActivationSigningMessage, + runtimePresenceProviderSigningMessage, +} from "cultnet-ts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { startFakeIdunnRudp, type FakeIdunnRudp } from "./fixtures/fake-idunn-rudp.js"; +import { idunnRuntimeFixtures } from "./fixtures/presence-descriptors.js"; + +const run = promisify(execFile); + +const GENERATION_UID = 61184; +const STATE_GID = 4242; +const ED25519_SPKI_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); +const PKCS8_ED25519_SEED_PREFIX = Buffer.from("302e020100300506032b657004220420", "hex"); + +// The launch needs root (to pick the uid and bind a machine-id) and a mount +// namespace (so the fixture machine-id does not replace the host's). The +// stopgap's node container has both; anywhere else the test steps aside. +const canLaunch = + process.getuid?.() === 0 && + (await run("unshare", ["--mount", "--", "true"]).then( + () => true, + () => false + )); + +/** + * The artifact under test. By default the esbuild bundle `pnpm bundle` makes, + * run by this process's node; a runbook proves a sealed release by naming its + * bundle and node here instead. + */ +const bundleOverride = process.env.HEIMDALL_BOOT_BUNDLE; +const nodeOverride = process.env.HEIMDALL_BOOT_NODE; + +async function freePort(): Promise { + return new Promise((resolve, reject) => { + const probe = net.createServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const { port } = probe.address() as net.AddressInfo; + probe.close(() => resolve(port)); + }); + }); +} + +interface Launch { + child: ChildProcess; + stderr: () => string; + exited: Promise; + candidatePort: number; +} + +describe.skipIf(!canLaunch)("the bundle booted as Idunn launches it", () => { + let base: string; + let bundle: string; + let node: string; + let idunn: FakeIdunnRudp; + let odin: dgram.Socket; + let odinPort: number; + const launched: ChildProcess[] = []; + + beforeAll(async () => { + base = await mkdtemp(path.join(os.tmpdir(), "heimdall-boot-")); + await chmod(base, 0o755); + if (bundleOverride) { + bundle = bundleOverride; + } else { + await run("corepack", ["pnpm", "bundle"], { cwd: path.join(import.meta.dirname, "..") }); + bundle = path.join(base, "heimdall.mjs"); + await copyFile(path.join(import.meta.dirname, "..", "dist", "heimdall"), bundle); + await chmod(bundle, 0o644); + } + node = nodeOverride ?? process.execPath; + idunn = await startFakeIdunnRudp(); + odin = dgram.createSocket("udp4"); + await new Promise((resolve) => odin.bind(0, "127.0.0.1", resolve)); + odinPort = odin.address().port; + }, 120_000); + + afterAll(() => { + for (const child of launched) child.kill("SIGKILL"); + idunn?.close(); + odin?.close(); + }); + + /** Credential files as Idunn holds them: readable only by the parent. */ + async function credentials(dir: string, identity: Buffer | undefined) { + await mkdir(dir, { recursive: true }); + const seed = path.join(dir, "activation-key.seed"); + await copyFile(path.join(idunnRuntimeFixtures, "activation-key.seed"), seed); + await chmod(seed, 0o400); + let identityFile: string | undefined; + if (identity) { + identityFile = path.join(dir, "provider-identity.credential"); + await writeFile(identityFile, identity, { mode: 0o400 }); + } + return { seed, identityFile }; + } + + /** Start the bundle as the generation unit does, with the fixture machine-id bound over the host's. */ + async function launch(label: string, identity: Buffer | undefined): Promise { + const dir = path.join(base, label); + await mkdir(dir, { recursive: true }); + await chmod(dir, 0o755); + + // Idunn's runtime bundle: the two records, read-only. + const bundleDir = path.join(dir, "bundle"); + await mkdir(bundleDir); + for (const name of ["expected.cc", "activation.cc"]) { + await copyFile(path.join(idunnRuntimeFixtures, name), path.join(bundleDir, name)); + await chmod(path.join(bundleDir, name), 0o444); + } + await chmod(bundleDir, 0o555); + + const machineId = path.join(dir, "machine-id"); + await copyFile(path.join(idunnRuntimeFixtures, "machine-id"), machineId); + await chmod(machineId, 0o444); + + const stateRoot = path.join(dir, "state"); + await mkdir(stateRoot); + await run("chown", [`0:${STATE_GID}`, stateRoot]); + await chmod(stateRoot, 0o2770); + + const heimdall = path.join(dir, "heimdall.mjs"); + await copyFile(bundle, heimdall); + await chmod(heimdall, 0o444); + + const files = await credentials(path.join(dir, "credentials"), identity); + const candidatePort = await freePort(); + + const fds = [openSync(files.seed, "r")]; + const names = ["gamecult-idunn-runtime-activation-key"]; + if (files.identityFile) { + fds.push(openSync(files.identityFile, "r")); + names.push("gamecult-runtime-presence-identity"); + } + + const env = { + PATH: process.env.PATH ?? "", + LISTEN_FDS: String(fds.length), + LISTEN_FDNAMES: names.join(":"), + GAMECULT_IDUNN_CANDIDATE_BIND: `127.0.0.1:${candidatePort}`, + GAMECULT_IDUNN_RUNTIME_BUNDLE: bundleDir, + GAMECULT_IDUNN_PROCESS_WRITE_LEASE: path.join(dir, "process-write-lease.cc"), + GC_ACCESS_BASE_URL: "https://heimdall.test", + GC_ACCESS_ISSUER: "https://heimdall.test", + GC_ACCESS_STORAGE_BACKEND: "memory", + GC_ACCESS_IDUNN_RUDP_HEALTH: `127.0.0.1:${idunn.port}`, + GC_ACCESS_IDUNN_HEALTH_CONTRACT: "heimdall.cultnet-rudp-provider-health", + GC_ACCESS_ODIN_CULTMESH_URI: `rudp://127.0.0.1:${odinPort}`, + CULTMESH_URI_ODIN_RUDP: `rudp://127.0.0.1:${odinPort}`, + GC_ACCESS_SIGNING_PRIVATE_KEY_BOOTSTRAP: "1", + GC_ACCESS_SIGNING_PRIVATE_KEY_PATH: path.join(stateRoot, "signing.pem"), + GC_ACCESS_SIGNING_KEY_ID: "boot-test", + }; + + // uid, gid and umask as Idunn's DynamicUser unit gives them. The fixture + // machine-id is bound only inside this mount namespace. + const script = + `mount --bind ${machineId} /etc/machine-id && ` + + `exec setpriv --reuid ${GENERATION_UID} --regid ${STATE_GID} --clear-groups ` + + `sh -c 'umask 0007; cd ${dir}; exec ${node} heimdall.mjs --state-root ${stateRoot}'`; + const child = spawn("unshare", ["--mount", "--", "sh", "-c", script], { + env, + stdio: ["ignore", "ignore", "pipe", ...fds], + }); + launched.push(child); + fds.forEach((fd) => closeSync(fd)); + + let stderr = ""; + child.stderr!.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + const exited = new Promise((resolve) => child.once("exit", (code) => resolve(code))); + return { child, stderr: () => stderr, exited, candidatePort }; + } + + async function answers(port: number): Promise { + return fetch(`http://127.0.0.1:${port}/.well-known/jwks.json`).then( + (response) => response.status, + () => undefined + ); + } + + it("signs warming presence on every pulse and answers HTTP", async () => { + const credential = await readFile(path.join(idunnRuntimeFixtures, "provider-identity.credential")); + const generation = await launch("signs", credential); + + // Startup publication, then one per 10 s pulse. + await Promise.race([ + idunn.frame(2), + generation.exited.then((code) => { + throw new Error(`bundle exited ${code}; stderr: ${generation.stderr()}`); + }), + ]); + + expect(await answers(generation.candidatePort)).toBe(200); + + // Public keys of the two proofs, from the fixtures' test keys. + const store = decode(credential) as unknown[][]; + const record = decode(store[0]![2] as Uint8Array) as unknown[]; + const providerKey = createPublicKey({ + key: Buffer.concat([ED25519_SPKI_PREFIX, Buffer.from(record[2] as number[])]), + format: "der", + type: "spki", + }); + const activationKey = createPublicKey( + createPrivateKey({ + key: Buffer.concat([ + PKCS8_ED25519_SEED_PREFIX, + await readFile(path.join(idunnRuntimeFixtures, "activation-key.seed")), + ]), + format: "der", + type: "pkcs8", + }) + ); + + let previousSequence = 0; + for (const frame of idunn.frames.slice(0, 3)) { + const message = parseCultNetMessage(decode(frame.payload), "cultnet.schema.v0"); + if (message.schemaVersion !== "cultnet.document_put_raw.v0") throw new Error("not a raw document put"); + const fields = decode(message.document.payload) as unknown[]; + expect(fields[RUNTIME_PRESENCE_SLOT.state]).toBe("warming"); + + const unsigned = [...fields]; + unsigned[RUNTIME_PRESENCE_SLOT.signature] = new Uint8Array(); + unsigned[RUNTIME_PRESENCE_SLOT.activationSignature] = new Uint8Array(); + const proof = encode(unsigned); + expect( + verify( + null, + runtimePresenceProviderSigningMessage(proof), + providerKey, + fields[RUNTIME_PRESENCE_SLOT.signature] as Uint8Array + ) + ).toBe(true); + expect( + verify( + null, + runtimePresenceActivationSigningMessage(proof), + activationKey, + fields[RUNTIME_PRESENCE_SLOT.activationSignature] as Uint8Array + ) + ).toBe(true); + + const sequence = fields[RUNTIME_PRESENCE_SLOT.publisherSequence] as number; + expect(sequence).toBeGreaterThan(previousSequence); + previousSequence = sequence; + } + + expect(generation.stderr()).not.toContain("failed on startup"); + expect(generation.stderr()).not.toContain("failed on interval"); + generation.child.kill("SIGKILL"); + }, 90_000); + + it.each([ + ["the identity descriptor is not passed", undefined, "gamecult-runtime-presence-identity"], + ["the identity is damaged", "damaged", "identity id check"], + ])("exits before listening when %s", async (_name, damage, expected) => { + const credential = await readFile(path.join(idunnRuntimeFixtures, "provider-identity.credential")); + const store = decode(credential) as unknown[][]; + const record = decode(store[0]![2] as Uint8Array) as unknown[]; + const publicKey = [...(record[2] as number[])]; + const original = { publicKeyHex: Buffer.from(publicKey).toString("hex"), id: record[1] as string, binding: record[5] as string }; + let identity: Buffer | undefined; + if (damage) { + publicKey[0] = publicKey[0]! ^ 1; + record[2] = publicKey; + identity = Buffer.from(encode([[store[0]![0], store[0]![1], encode(record), store[0]![3], store[0]![4]]])); + } + const started = Date.now(); + const generation = await launch(damage ? "damaged" : "no-identity", identity); + + const code = await Promise.race([ + generation.exited, + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 5_000)), + ]); + + expect(code, generation.stderr()).not.toBe("timeout"); + expect(code).not.toBe(0); + expect(Date.now() - started).toBeLessThan(6_000); + expect(await answers(generation.candidatePort)).toBeUndefined(); + expect(generation.stderr()).toContain(expected); + // No byte of the credential: not the key bytes, not the id, not the binding. + expect(generation.stderr()).not.toContain(original.publicKeyHex); + expect(generation.stderr()).not.toContain(original.id.slice(0, 16)); + expect(generation.stderr()).not.toContain(original.binding.slice(-16)); + }, 30_000); +}); diff --git a/tests/presence-identity-descriptor.test.ts b/tests/presence-identity-descriptor.test.ts index 731d16e..d01797b 100644 --- a/tests/presence-identity-descriptor.test.ts +++ b/tests/presence-identity-descriptor.test.ts @@ -1,18 +1,17 @@ import { access, mkdtemp, readFile, writeFile } from "node:fs/promises"; -import { openSync, closeSync } from "node:fs"; +import { openSync } from "node:fs"; import os from "node:os"; import path from "node:path"; -import { encode } from "@msgpack/msgpack"; -import { SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; +import { decode, encode } from "@msgpack/msgpack"; import { describe, expect, it } from "vitest"; import { - openOrEnrollProviderHealthIdentity, openProviderHealthIdentity, passedDescriptor, providerHealthPublicIdentity, } from "../src/provider-health-identity.js"; +import { fixtureMachineId, idunnRuntimeFixtures } from "./fixtures/presence-descriptors.js"; const machineIdAvailable = await readFile("/etc/machine-id", "utf8").then( () => true, @@ -23,59 +22,79 @@ const machineIdAvailable = await readFile("/etc/machine-id", "utf8").then( * The identity is machine-id bound, so enrolment only works where a machine-id * exists. These run on Linux and are skipped elsewhere. */ -describe.skipIf(!machineIdAvailable)("presence identity from a passed descriptor", () => { - it("reads an Idunn-written store, not a cultcache-ts one", async () => { +/** Name the presence descriptor so it lands on whatever fd open() returned. */ +function presenceEnv(fd: number): NodeJS.ProcessEnv { + // The lookup maps a name index onto 3 + index, so the name list is padded + // until the presence entry lands on the descriptor open() actually returned. + // Assuming it is 3 is how this passes on one machine and reads the wrong + // descriptor on another. + const names = [ + ...Array.from({ length: fd - 3 }, (_, i) => `pad-${i}`), + "gamecult-runtime-presence-identity", + ].join(":"); + return { LISTEN_FDNAMES: names, LISTEN_PID: String(process.pid) }; +} + +describe("presence identity from a passed descriptor", () => { + it("decodes the Rust owner's record, whose key fields are integer arrays", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-presence-")); + const fd = openSync(path.join(idunnRuntimeFixtures, "provider-identity.credential"), "r"); - // Enrol with the TypeScript client to get a genuine machine-bound payload, - // then re-wrap that payload the way cultcache-rs writes a store: a bare - // array of positional [key, type, payload, stored_at, schema_id] envelopes. - // Idunn writes with the Rust client, and cultcache-ts cannot read that - // layout, so wrapping it here is what makes this test resemble reality. - const tsStore = path.join(dir, "ts-enrolled.cc"); - const enrolled = await openOrEnrollProviderHealthIdentity(tsStore); - const records = await new SingleFileMessagePackBackingStore(tsStore).pullAll(); - expect(records).toHaveLength(1); - - const idunnStore = path.join(dir, "runtime-presence-identity.cc"); - await writeFile( - idunnStore, - encode([ - [ - "gamecult-provider-health-identity", - "gamecult.provider_health_identity.private.v1", - records[0]!.payload, - "2026-09-06T00:00:00.000Z", - "gamecult.provider_health_identity.private.v1", - ], - ]) + const opened = await openProviderHealthIdentity( + path.join(dir, "would-be-self-enrolled.cc"), + presenceEnv(fd), + fixtureMachineId ); - const fd = openSync(idunnStore, "r"); - try { - // The lookup maps a name index onto 3 + index, so the name list is padded - // until the presence entry lands on whatever descriptor open() actually - // returned. Assuming it is 3 is how this passes on one machine and reads - // the wrong descriptor on another. - const names = [ - ...Array.from({ length: fd - 3 }, (_, i) => `pad-${i}`), - "gamecult-runtime-presence-identity", - ].join(":"); + // The fixture's own public key means it came from the descriptor. A fresh + // self-enrolment would produce a different key, which is exactly the failure + // that makes Idunn refuse this daemon's health and stall promotion. + const store = decode(await readFile(path.join(idunnRuntimeFixtures, "provider-identity.credential"))) as unknown[][]; + const record = decode(store[0]![2] as Uint8Array) as unknown[]; + expect(providerHealthPublicIdentity(opened).publicKeyHex).toBe( + Buffer.from(record[2] as number[]).toString("hex") + ); + await expect(access(path.join(dir, "would-be-self-enrolled.cc"))).rejects.toMatchObject({ code: "ENOENT" }); + }); - const opened = await openProviderHealthIdentity( - path.join(dir, "would-be-self-enrolled.cc"), - { LISTEN_FDNAMES: names, LISTEN_PID: String(process.pid) } - ); + it("refuses a bin-shaped record on the descriptor path, naming the key-shape check and no value", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-presence-bin-")); + const store = decode(await readFile(path.join(idunnRuntimeFixtures, "provider-identity.credential"))) as unknown[][]; + const record = decode(store[0]![2] as Uint8Array) as unknown[]; + // What this process's own enroller writes: the same record with bin key fields. + const binRecord = record.map((field, index) => + index === 2 || index === 3 ? Uint8Array.from(field as number[]) : field + ); + const binStore = path.join(dir, "bin-shaped.cc"); + await writeFile(binStore, encode([[store[0]![0], store[0]![1], encode(binRecord), store[0]![3], store[0]![4]]])); + const fd = openSync(binStore, "r"); - // Same key means it came from the descriptor. A fresh self-enrolment - // would produce a different key -- which is exactly the failure that - // makes Idunn refuse this daemon's health and stall promotion. - expect(providerHealthPublicIdentity(opened).publicKeyHex).toBe( - providerHealthPublicIdentity(enrolled).publicKeyHex - ); - } finally { - closeSync(fd); - } + const failure = await openProviderHealthIdentity(path.join(dir, "x.cc"), presenceEnv(fd), fixtureMachineId).then( + () => undefined, + (error: Error) => error + ); + + expect(failure?.message).toContain("key shape check"); + expect(failure?.message).not.toContain((record[1] as string).slice(0, 16)); + }); + + it.each([ + ["schema", 0, "bogus", "schema check"], + ["protector", 4, "bogus", "protector check"], + ["binding", 5, "bogus", "binding check"], + ["version", 6, "v9", "version check"], + ])("a record failing only the %s check is refused by that check", async (_name, index, replacement, check) => { + const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-presence-check-")); + const store = decode(await readFile(path.join(idunnRuntimeFixtures, "provider-identity.credential"))) as unknown[][]; + const record = decode(store[0]![2] as Uint8Array) as unknown[]; + record[index] = replacement; + const damaged = path.join(dir, "damaged.cc"); + await writeFile(damaged, encode([[store[0]![0], store[0]![1], encode(record), store[0]![3], store[0]![4]]])); + const fd = openSync(damaged, "r"); + + await expect( + openProviderHealthIdentity(path.join(dir, "x.cc"), presenceEnv(fd), fixtureMachineId) + ).rejects.toThrow(check); }); }); diff --git a/tests/publisher-sequence.test.ts b/tests/publisher-sequence.test.ts index 9a4f147..efded2c 100644 --- a/tests/publisher-sequence.test.ts +++ b/tests/publisher-sequence.test.ts @@ -1,17 +1,12 @@ -import { mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; +import { mkdtemp, readdir, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { loadConfig } from "../src/config.js"; -import { PublisherSequence, buildPresenceStatement } from "../src/runtime-presence.js"; -import { bundleFacts, openPresenceDescriptors, writePresenceFixtures } from "./fixtures/presence-descriptors.js"; - -const machineIdAvailable = await readFile("/etc/machine-id", "utf8").then( - () => true, - () => false -); +import { PublisherSequence, openPresenceAuthority } from "../src/runtime-presence.js"; +import { fixtureMachineId, openPresenceDescriptors, writePresenceFixtures } from "./fixtures/presence-descriptors.js"; describe("PublisherSequence", () => { it("starts at the observed millis", () => { @@ -51,10 +46,11 @@ describe("PublisherSequence", () => { }); }); -describe.skipIf(!machineIdAvailable)("presence publication and the state root", () => { - it("touches no path under dataRoot", async () => { +describe("the presence authority", () => { + it("opens once, touches no path under dataRoot, and signs every pulse after its descriptors are gone", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "heimdall-nodataroot-")); - await writePresenceFixtures(path.join(dir, "fixtures")); + const fixtures = path.join(dir, "fixtures"); + await writePresenceFixtures(fixtures); // A regular file as dataRoot: opening anything beneath it fails with // ENOTDIR for every uid, root included. const dataRoot = path.join(dir, "not-a-directory"); @@ -64,21 +60,21 @@ describe.skipIf(!machineIdAvailable)("presence publication and the state root", GC_ACCESS_DATA_ROOT: dataRoot, }); - const descriptors = openPresenceDescriptors(path.join(dir, "fixtures")); + const descriptors = openPresenceDescriptors(fixtures); try { + const authority = await openPresenceAuthority( + { ...config, idunnRuntimeBundlePath: fixtures }, + descriptors.env, + fixtureMachineId, + ); + // The descriptors were read whole and closed: a second read would find + // nothing, so signing again proves the authority holds what it decoded. const observedAtUnixMillis = Date.parse("2026-10-23T12:00:00.000Z"); - const statement = await buildPresenceStatement({ - bundle: bundleFacts, - state: "warming", - detail: "probe", - boundEndpoint: null, - writeLeaseSha256: null, - observedAtUnixMillis, - publisherSequence: new PublisherSequence().next(observedAtUnixMillis), - providerHealthIdentityPath: config.providerHealthIdentityPath, - env: descriptors.env, - }); - expect(statement.target).toBe("heimdall"); + const first = authority.sign("warming", "probe", observedAtUnixMillis); + const second = authority.sign("warming", "probe", observedAtUnixMillis); + expect(first.target).toBe("heimdall"); + expect(second.target).toBe("heimdall"); + expect(Buffer.from(second.payload).equals(Buffer.from(first.payload))).toBe(false); } finally { descriptors.close(); } From 1a36175f3a0ab3b66313ae82c8b5a21d5b96c939 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 17:49:23 +0200 Subject: [PATCH 84/85] Presence authority test: the Rust fixture's target is the fixture's own, not heimdall Co-Authored-By: Claude Opus 5.5 --- tests/publisher-sequence.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/publisher-sequence.test.ts b/tests/publisher-sequence.test.ts index efded2c..cfddac2 100644 --- a/tests/publisher-sequence.test.ts +++ b/tests/publisher-sequence.test.ts @@ -72,8 +72,8 @@ describe("the presence authority", () => { const observedAtUnixMillis = Date.parse("2026-10-23T12:00:00.000Z"); const first = authority.sign("warming", "probe", observedAtUnixMillis); const second = authority.sign("warming", "probe", observedAtUnixMillis); - expect(first.target).toBe("heimdall"); - expect(second.target).toBe("heimdall"); + expect(first.target).toBe("streampixels-web"); + expect(second.target).toBe("streampixels-web"); expect(Buffer.from(second.payload).equals(Buffer.from(first.payload))).toBe(false); } finally { descriptors.close(); From 5146c722afd8c79414f06ab6b20d352fbb1c6d70 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 17:50:37 +0200 Subject: [PATCH 85/85] Boot test: fail on a logged publication fault, and require presence at startup A silent early return in the startup publication would otherwise pass, the 10 s pulse loop filling in the frames later. Co-Authored-By: Claude Opus 5.5 --- tests/idunn-boot.test.ts | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/idunn-boot.test.ts b/tests/idunn-boot.test.ts index d758910..a645986 100644 --- a/tests/idunn-boot.test.ts +++ b/tests/idunn-boot.test.ts @@ -199,13 +199,31 @@ describe.skipIf(!canLaunch)("the bundle booted as Idunn launches it", () => { const credential = await readFile(path.join(idunnRuntimeFixtures, "provider-identity.credential")); const generation = await launch("signs", credential); - // Startup publication, then one per 10 s pulse. + // Startup publication, then one per 10 s pulse. A publication fault is + // reported as it is logged rather than after the frames never arrive. + const publicationFault = new Promise((_resolve, reject) => { + const watch = setInterval(() => { + if (generation.stderr().includes(" failed on ")) { + clearInterval(watch); + reject(new Error(`publication failed; stderr: ${generation.stderr()}`)); + } + }, 200); + void idunn.frame(2).then(() => clearInterval(watch)); + }); + const exitedEarly = generation.exited.then((code) => { + throw new Error(`bundle exited ${code}; stderr: ${generation.stderr()}`); + }); + // The first statement is the startup publication, which follows the listen + // directly; the 10 s pulse loop has not yet fired. await Promise.race([ - idunn.frame(2), - generation.exited.then((code) => { - throw new Error(`bundle exited ${code}; stderr: ${generation.stderr()}`); - }), + idunn.frame(0), + publicationFault, + exitedEarly, + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error(`no presence at startup; stderr: ${generation.stderr()}`)), 8_000) + ), ]); + await Promise.race([idunn.frame(2), publicationFault, exitedEarly]); expect(await answers(generation.candidatePort)).toBe(200);