From 8541ae3f2f9ee2866851758577dc83a67d72e811 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:15:34 +0200 Subject: [PATCH] Startup proves the database can take Heimdall's writes Soul pass 4, F4: the startup check passed on databases where every request still failed: a role granted SELECT only (O2), and a table whose primary key and key column were dropped (N), including on a start that applied the schema, since CREATE ... IF NOT EXISTS leaves a damaged table as it is. Every start now ends in checkSchema, after applying the schema or not. It requires, read from the schema SQL: every relation; every column the created tables declare, not only the added one (read from pg_attribute, so a column the role may not see is not reported as missing); every primary key and unique constraint as a non-partial unique index on exactly its columns, which is what the store's ON CONFLICT clauses and the foreign keys rely on; and SELECT, INSERT and UPDATE on every table. A missing piece refuses with SCHEMA_MISSING, a missing privilege with SCHEMA_PRIVILEGES, through the same fixed startup message. A live-Postgres block drives createStore itself, skipped unless HEIMDALL_TEST_PG_ADMIN_URL names a role that may create databases and roles: a read-only role (and its canary password stays out of the message), a dropped primary key with and without applying, a unique key replaced by a partial one, and a dropped column. Co-Authored-By: Claude Opus 5.5 --- src/store/index.ts | 11 ++- src/store/postgres.ts | 65 +++++++++++++---- src/store/schema.ts | 59 +++++++++++++-- tests/store.test.ts | 166 +++++++++++++++++++++++++++++++++++++----- 4 files changed, 261 insertions(+), 40 deletions(-) diff --git a/src/store/index.ts b/src/store/index.ts index b281ec4..32f6e8e 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -14,10 +14,15 @@ export async function createStore(config: HeimdallConfig): Promise { - const result = await this.pool.query<{ missing: string }>( + const result = await this.pool.query<{ kind: "missing" | "privilege" }>( ` - SELECT name AS missing FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL + SELECT 'missing' AS kind FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL UNION ALL - SELECT required.table_name || '.' || required.column_name - FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name) + SELECT 'missing' FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name) WHERE NOT EXISTS ( - SELECT 1 FROM information_schema.columns c - WHERE c.table_schema = current_schema() - AND c.table_name = required.table_name - AND c.column_name = required.column_name + SELECT 1 FROM pg_attribute a + WHERE a.attrelid = to_regclass(required.table_name) + AND a.attname = required.column_name + AND a.attnum > 0 + AND NOT a.attisdropped ) + UNION ALL + SELECT 'missing' FROM unnest($4::text[], $5::text[], $6::boolean[]) AS required(table_name, columns, is_primary) + WHERE NOT EXISTS ( + SELECT 1 FROM pg_index i + WHERE i.indrelid = to_regclass(required.table_name) + AND i.indisunique + AND i.indpred IS NULL + AND (i.indisprimary OR NOT required.is_primary) + AND i.indnatts = cardinality(string_to_array(required.columns, ',')) + AND ( + SELECT string_agg(a.attname, ',' ORDER BY a.attname) + FROM unnest(i.indkey::int2[]) AS key(attnum) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum + ) = required.columns + ) + UNION ALL + SELECT 'privilege' FROM unnest($7::text[]) AS required(table_name) + CROSS JOIN unnest($8::text[]) AS wanted(privilege) + WHERE to_regclass(required.table_name) IS NOT NULL + AND NOT has_table_privilege(to_regclass(required.table_name), wanted.privilege) `, [ REQUIRED_RELATIONS, REQUIRED_COLUMNS.map((required) => required.table), REQUIRED_COLUMNS.map((required) => required.column), + REQUIRED_KEYS.map((required) => required.table), + REQUIRED_KEYS.map((required) => required.columns), + REQUIRED_KEYS.map((required) => required.primary), + REQUIRED_TABLES, + REQUIRED_TABLE_PRIVILEGES, ] ); - if (result.rows.length > 0) { + if (result.rows.some((row) => row.kind === "missing")) { throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); } + if (result.rows.length > 0) { + throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { + code: "SCHEMA_PRIVILEGES", + }); + } } async close(): Promise { diff --git a/src/store/schema.ts b/src/store/schema.ts index 8e9c921..cb80eb0 100644 --- a/src/store/schema.ts +++ b/src/store/schema.ts @@ -163,15 +163,60 @@ CREATE INDEX IF NOT EXISTS audit_events_lookup_idx ON audit_events(account_id, session_id, app_slug, created_at); `; -/** - * What a start that does not apply the schema must find, read from the SQL - * above so the two cannot drift: every table and index it creates, and every - * column it adds to an existing table. +const schemaText = CREATE_SCHEMA_SQL.replace(/\r\n/g, "\n"); +const createdTables = [...schemaText.matchAll(/^CREATE TABLE IF NOT EXISTS (\w+) \(\n([\s\S]*?)\n\);/gm)].map((match) => ({ + table: match[1]!, + lines: match[2]!.split("\n").map((line) => line.trim().replace(/,$/, "")), +})); + +/** The sorted, comma-separated columns of a `PRIMARY KEY (a, b)` or `UNIQUE(a, b)` line. */ +function keyColumns(line: string, keyword: "PRIMARY KEY" | "UNIQUE"): string | undefined { + const match = new RegExp(`^${keyword}\\s*\\(([^)]*)\\)`).exec(line); + return match ? match[1]!.split(",").map((column) => column.trim()).sort().join(",") : undefined; +} + +/* + * What every start must find before it serves, read from the SQL above so the + * two cannot drift. A start that applies the schema checks as well, because + * CREATE ... IF NOT EXISTS leaves a damaged table as it is. */ + +/** Every table and index the schema creates. */ export const REQUIRED_RELATIONS = [ - ...CREATE_SCHEMA_SQL.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), + ...schemaText.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm), ].map((match) => match[1]!); +/** Every table the schema creates. The store reads and writes each one. */ +export const REQUIRED_TABLES = createdTables.map(({ table }) => table); + +/** What the store does to its tables: it reads, inserts and updates, and never deletes. */ +export const REQUIRED_TABLE_PRIVILEGES = ["SELECT", "INSERT", "UPDATE"]; + +/** Every column a created table declares, and every column the schema adds to an existing table. */ export const REQUIRED_COLUMNS = [ - ...CREATE_SCHEMA_SQL.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm), -].map((match) => ({ table: match[1]!, column: match[2]! })); + ...createdTables.flatMap(({ table, lines }) => + lines + .filter((line) => !/^(?:PRIMARY KEY|UNIQUE)\b/.test(line)) + .map((line) => ({ table, column: /^(\w+)/.exec(line)![1]! })) + ), + ...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm)].map((match) => ({ + table: match[1]!, + column: match[2]!, + })), +]; + +/** + * Every primary key and unique constraint a created table declares, by its + * sorted columns. Foreign keys reference the primary keys, and each of the + * store's ON CONFLICT clauses needs a unique index on exactly its columns: + * without one, every such write fails. + */ +export const REQUIRED_KEYS = createdTables.flatMap(({ table, lines }) => + lines.flatMap((line) => { + const inline = /^(\w+) .*\bPRIMARY KEY\b/.exec(line); + const columns = inline?.[1] ?? keyColumns(line, "PRIMARY KEY"); + if (columns) return [{ table, columns, primary: true }]; + const unique = keyColumns(line, "UNIQUE"); + return unique ? [{ table, columns: unique, primary: false }] : []; + }) +); diff --git a/tests/store.test.ts b/tests/store.test.ts index c28e353..5f9f5a5 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -1,7 +1,17 @@ -import { describe, expect, it } from "vitest"; +import { randomUUID } from "node:crypto"; +import { Pool } from "pg"; +import { afterEach, describe, expect, it } from "vitest"; import { newDb } from "pg-mem"; +import { type HeimdallConfig } from "../src/config.js"; +import { createStore } from "../src/store/index.js"; import { createPostgresStore, PostgresStore } from "../src/store/postgres.js"; -import { REQUIRED_COLUMNS, REQUIRED_RELATIONS } from "../src/store/schema.js"; +import { + REQUIRED_COLUMNS, + REQUIRED_KEYS, + REQUIRED_RELATIONS, + REQUIRED_TABLE_PRIVILEGES, + REQUIRED_TABLES, +} from "../src/store/schema.js"; describe("PostgresStore", () => { it("round-trips core auth records through postgres", async () => { @@ -212,33 +222,155 @@ describe("createPostgresStore", () => { }); describe("PostgresStore.checkSchema", () => { - it("requires every table and index the schema creates and every column it adds", () => { + it("requires every table, index, column and key the schema makes", () => { expect(REQUIRED_RELATIONS).toEqual( expect.arrayContaining(["accounts", "audit_events", "auth_completions_attempt_unconsumed_unique_idx", "audit_events_lookup_idx"]) ); expect(REQUIRED_RELATIONS).toHaveLength(17); - expect(REQUIRED_COLUMNS).toEqual([{ table: "auth_completions", column: "attempt_id" }]); + expect(REQUIRED_TABLES).toEqual([ + "accounts", + "linked_identities", + "sessions", + "auth_attempts", + "private_command_receipts", + "auth_completions", + "capability_grants", + "entitlement_snapshots", + "audit_events", + ]); + expect(REQUIRED_COLUMNS).toHaveLength(85); + expect(REQUIRED_COLUMNS).toEqual( + expect.arrayContaining([ + { table: "auth_completions", column: "code" }, + { table: "auth_completions", column: "mode" }, + { table: "linked_identities", column: "profile_json" }, + { table: "auth_completions", column: "attempt_id" }, + ]) + ); + // The keys the store's ON CONFLICT clauses and the foreign keys rely on. + expect(REQUIRED_KEYS).toEqual( + expect.arrayContaining([ + { table: "linked_identities", columns: "provider,provider_user_id", primary: false }, + { table: "sessions", columns: "id", primary: true }, + { table: "private_command_receipts", columns: "app_slug,idempotency_key", primary: true }, + { table: "entitlement_snapshots", columns: "account_id,provider,scope", primary: false }, + { table: "accounts", columns: "id", primary: true }, + { table: "auth_completions", columns: "code", primary: true }, + ]) + ); + expect(REQUIRED_KEYS).toHaveLength(11); + expect(REQUIRED_TABLE_PRIVILEGES).toEqual(["SELECT", "INSERT", "UPDATE"]); }); - function storeAnswering(missing: string[]) { - const queries: unknown[][] = []; - const store = new PostgresStore({ - query: (async (_text: string, values: unknown[]) => { - queries.push(values); - return { rows: missing.map((name) => ({ missing: name })) }; - }) as never, + function storeAnswering(kinds: string[]) { + return new PostgresStore({ + query: (async () => ({ rows: kinds.map((kind) => ({ kind })) })) as never, end: async () => undefined, }); - return { store, queries }; } - it("refuses with SCHEMA_MISSING when anything is missing", async () => { - const { store, queries } = storeAnswering(["auth_completions.attempt_id"]); - await expect(store.checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" }); - expect(queries).toEqual([[REQUIRED_RELATIONS, ["auth_completions"], ["attempt_id"]]]); + it("refuses with SCHEMA_MISSING when anything is missing, whatever else is", async () => { + await expect(storeAnswering(["privilege", "missing"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" }); + }); + + it("refuses with SCHEMA_PRIVILEGES when only a privilege is missing", async () => { + await expect(storeAnswering(["privilege"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_PRIVILEGES" }); }); it("passes when nothing is missing", async () => { - await expect(storeAnswering([]).store.checkSchema()).resolves.toBeUndefined(); + await expect(storeAnswering([]).checkSchema()).resolves.toBeUndefined(); + }); +}); + +// Against a real Postgres, through the startup path itself: createStore, with +// and without applying the schema. HEIMDALL_TEST_PG_ADMIN_URL names a role +// that may create databases and roles; each case gets its own database. +const adminUrl = process.env.HEIMDALL_TEST_PG_ADMIN_URL; +describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_ADMIN_URL)", () => { + const refusal = (code: string) => `Postgres storage could not be prepared (${code}); check GC_ACCESS_DATABASE_URL_FILE.`; + const cleanup: Array<() => Promise> = []; + afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); + }); + + function urlFor(database: string, user?: string, password?: string): string { + const url = new URL(adminUrl!); + url.pathname = `/${database}`; + if (user) url.username = user; + if (password) url.password = password; + return url.toString(); + } + + /** A fresh database with the schema applied by its owner, and an owner pool on it. */ + async function appliedDatabase(): Promise<{ name: string; pool: Pool }> { + const name = `heimdall_check_${randomUUID().replace(/-/g, "")}`; + const admin = new Pool({ connectionString: adminUrl }); + await admin.query(`CREATE DATABASE ${name}`); + cleanup.push(async () => { + await admin.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`); + await admin.end(); + }); + const pool = new Pool({ connectionString: urlFor(name) }); + cleanup.push(async () => { + await pool.end(); + }); + await new PostgresStore(pool).ensureSchema(); + return { name, pool }; + } + + async function start(url: string, applySchemaOnStartup: boolean): Promise { + try { + const store = await createStore({ storage: { backend: "postgres", databaseUrl: url, applySchemaOnStartup } } as HeimdallConfig); + await store.close(); + return "started"; + } catch (error) { + return (error as Error).message; + } + } + + it("starts on an applied database, applying or not", async () => { + const { name } = await appliedDatabase(); + expect(await start(urlFor(name), false)).toBe("started"); + expect(await start(urlFor(name), true)).toBe("started"); + }); + + it("refuses a role that may only read, by code and without its password", async () => { + const { name, pool } = await appliedDatabase(); + const role = `heimdall_ro_${randomUUID().replace(/-/g, "").slice(0, 12)}`; + await pool.query(`CREATE ROLE ${role} LOGIN PASSWORD 'CANARYrolepw'`); + cleanup.push(async () => { + await pool.query(`REVOKE ALL ON ALL TABLES IN SCHEMA public FROM ${role}`); + await pool.query(`DROP ROLE IF EXISTS ${role}`); + }); + await pool.query(`GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${role}`); + + const message = await start(urlFor(name, role, "CANARYrolepw"), false); + expect(message).toBe(refusal("SCHEMA_PRIVILEGES")); + expect(message).not.toContain("CANARY"); + }); + + it("refuses a table whose primary key was dropped, even after applying the schema", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE auth_completions DROP CONSTRAINT auth_completions_pkey"); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + expect(await start(urlFor(name), true)).toBe(refusal("SCHEMA_MISSING")); + }); + + it("refuses a unique key the store's ON CONFLICT needs, replaced by a partial one", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key"); + await pool.query( + "CREATE UNIQUE INDEX entitlement_snapshots_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed" + ); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); + }); + + it("refuses a created table's dropped column", async () => { + const { name, pool } = await appliedDatabase(); + await pool.query("ALTER TABLE auth_completions DROP COLUMN mode"); + + expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING")); }); });