Startup proves the database can take Heimdall's writes

Soul pass 4, F4: the startup check passed on databases where every request
still failed: a role granted SELECT only (O2), and a table whose primary
key and key column were dropped (N), including on a start that applied the
schema, since CREATE ... IF NOT EXISTS leaves a damaged table as it is.

Every start now ends in checkSchema, after applying the schema or not. It
requires, read from the schema SQL: every relation; every column the
created tables declare, not only the added one (read from pg_attribute, so
a column the role may not see is not reported as missing); every primary
key and unique constraint as a non-partial unique index on exactly its
columns, which is what the store's ON CONFLICT clauses and the foreign
keys rely on; and SELECT, INSERT and UPDATE on every table. A missing
piece refuses with SCHEMA_MISSING, a missing privilege with
SCHEMA_PRIVILEGES, through the same fixed startup message.

A live-Postgres block drives createStore itself, skipped unless
HEIMDALL_TEST_PG_ADMIN_URL names a role that may create databases and
roles: a read-only role (and its canary password stays out of the
message), a dropped primary key with and without applying, a unique key
replaced by a partial one, and a dropped column.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-30 17:15:34 +02:00
commit 8541ae3f2f
4 changed files with 261 additions and 40 deletions

View file

@ -14,10 +14,15 @@ export async function createStore(config: HeimdallConfig): Promise<HeimdallStore
const store = createPostgresStore(config.storage.databaseUrl ?? "postgres://127.0.0.1/heimdall");
try {
// Every start connects, and one that does not apply the schema proves it
// is there, so a misbound URL or an empty database fails here rather than
// Every start connects and proves the database can take its requests,
// whether or not it applied the schema first (CREATE ... IF NOT EXISTS
// leaves a damaged table as it is), so a misbound URL, an empty or
// damaged database, or a role that may not write fails here rather than
// in the first request.
await (config.storage.applySchemaOnStartup ? store.ensureSchema() : store.checkSchema());
if (config.storage.applySchemaOnStartup) {
await store.ensureSchema();
}
await store.checkSchema();
} catch (error) {
// pg's and Node's errors name the host, the database or the URL itself,
// and any of those can hold part of a secret that was bound or escaped

View file

@ -1,7 +1,14 @@
import { randomUUID } from "node:crypto";
import { Pool, type PoolClient, type QueryResultRow } from "pg";
import { type AppSlug, type HeimdallAuthAttemptStatus, type LinkedIdentityInput, type Provider } from "../contracts.js";
import { CREATE_SCHEMA_SQL, REQUIRED_COLUMNS, REQUIRED_RELATIONS } from "./schema.js";
import {
CREATE_SCHEMA_SQL,
REQUIRED_COLUMNS,
REQUIRED_KEYS,
REQUIRED_RELATIONS,
REQUIRED_TABLE_PRIVILEGES,
REQUIRED_TABLES,
} from "./schema.js";
import {
type CreateAccountInput,
type CreateAuthAttemptInput,
@ -292,33 +299,65 @@ export class PostgresStore implements HeimdallStore {
}
/**
* Proves, for a start that does not apply the schema, that the URL connects
* and that the database holds every relation and column the schema makes.
* A missing one fails with code SCHEMA_MISSING.
* Proves, before Heimdall serves, that the database can take its requests:
* every relation, column and key the schema makes is there, and the role
* may read, insert and update every table. A missing piece fails with code
* SCHEMA_MISSING and a missing privilege with SCHEMA_PRIVILEGES; neither
* error names the piece.
*/
async checkSchema(): Promise<void> {
const result = await this.pool.query<{ missing: string }>(
const result = await this.pool.query<{ kind: "missing" | "privilege" }>(
`
SELECT name AS missing FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL
SELECT 'missing' AS kind FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL
UNION ALL
SELECT required.table_name || '.' || required.column_name
FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name)
SELECT 'missing' FROM unnest($2::text[], $3::text[]) AS required(table_name, column_name)
WHERE NOT EXISTS (
SELECT 1 FROM information_schema.columns c
WHERE c.table_schema = current_schema()
AND c.table_name = required.table_name
AND c.column_name = required.column_name
SELECT 1 FROM pg_attribute a
WHERE a.attrelid = to_regclass(required.table_name)
AND a.attname = required.column_name
AND a.attnum > 0
AND NOT a.attisdropped
)
UNION ALL
SELECT 'missing' FROM unnest($4::text[], $5::text[], $6::boolean[]) AS required(table_name, columns, is_primary)
WHERE NOT EXISTS (
SELECT 1 FROM pg_index i
WHERE i.indrelid = to_regclass(required.table_name)
AND i.indisunique
AND i.indpred IS NULL
AND (i.indisprimary OR NOT required.is_primary)
AND i.indnatts = cardinality(string_to_array(required.columns, ','))
AND (
SELECT string_agg(a.attname, ',' ORDER BY a.attname)
FROM unnest(i.indkey::int2[]) AS key(attnum)
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum
) = required.columns
)
UNION ALL
SELECT 'privilege' FROM unnest($7::text[]) AS required(table_name)
CROSS JOIN unnest($8::text[]) AS wanted(privilege)
WHERE to_regclass(required.table_name) IS NOT NULL
AND NOT has_table_privilege(to_regclass(required.table_name), wanted.privilege)
`,
[
REQUIRED_RELATIONS,
REQUIRED_COLUMNS.map((required) => required.table),
REQUIRED_COLUMNS.map((required) => required.column),
REQUIRED_KEYS.map((required) => required.table),
REQUIRED_KEYS.map((required) => required.columns),
REQUIRED_KEYS.map((required) => required.primary),
REQUIRED_TABLES,
REQUIRED_TABLE_PRIVILEGES,
]
);
if (result.rows.length > 0) {
if (result.rows.some((row) => row.kind === "missing")) {
throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" });
}
if (result.rows.length > 0) {
throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), {
code: "SCHEMA_PRIVILEGES",
});
}
}
async close(): Promise<void> {

View file

@ -163,15 +163,60 @@ CREATE INDEX IF NOT EXISTS audit_events_lookup_idx
ON audit_events(account_id, session_id, app_slug, created_at);
`;
/**
* What a start that does not apply the schema must find, read from the SQL
* above so the two cannot drift: every table and index it creates, and every
* column it adds to an existing table.
const schemaText = CREATE_SCHEMA_SQL.replace(/\r\n/g, "\n");
const createdTables = [...schemaText.matchAll(/^CREATE TABLE IF NOT EXISTS (\w+) \(\n([\s\S]*?)\n\);/gm)].map((match) => ({
table: match[1]!,
lines: match[2]!.split("\n").map((line) => line.trim().replace(/,$/, "")),
}));
/** The sorted, comma-separated columns of a `PRIMARY KEY (a, b)` or `UNIQUE(a, b)` line. */
function keyColumns(line: string, keyword: "PRIMARY KEY" | "UNIQUE"): string | undefined {
const match = new RegExp(`^${keyword}\\s*\\(([^)]*)\\)`).exec(line);
return match ? match[1]!.split(",").map((column) => column.trim()).sort().join(",") : undefined;
}
/*
* What every start must find before it serves, read from the SQL above so the
* two cannot drift. A start that applies the schema checks as well, because
* CREATE ... IF NOT EXISTS leaves a damaged table as it is.
*/
/** Every table and index the schema creates. */
export const REQUIRED_RELATIONS = [
...CREATE_SCHEMA_SQL.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm),
...schemaText.matchAll(/^CREATE (?:UNIQUE )?(?:TABLE|INDEX) IF NOT EXISTS (\w+)/gm),
].map((match) => match[1]!);
/** Every table the schema creates. The store reads and writes each one. */
export const REQUIRED_TABLES = createdTables.map(({ table }) => table);
/** What the store does to its tables: it reads, inserts and updates, and never deletes. */
export const REQUIRED_TABLE_PRIVILEGES = ["SELECT", "INSERT", "UPDATE"];
/** Every column a created table declares, and every column the schema adds to an existing table. */
export const REQUIRED_COLUMNS = [
...CREATE_SCHEMA_SQL.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm),
].map((match) => ({ table: match[1]!, column: match[2]! }));
...createdTables.flatMap(({ table, lines }) =>
lines
.filter((line) => !/^(?:PRIMARY KEY|UNIQUE)\b/.test(line))
.map((line) => ({ table, column: /^(\w+)/.exec(line)![1]! }))
),
...[...schemaText.matchAll(/^ALTER TABLE (\w+) ADD COLUMN IF NOT EXISTS (\w+)/gm)].map((match) => ({
table: match[1]!,
column: match[2]!,
})),
];
/**
* Every primary key and unique constraint a created table declares, by its
* sorted columns. Foreign keys reference the primary keys, and each of the
* store's ON CONFLICT clauses needs a unique index on exactly its columns:
* without one, every such write fails.
*/
export const REQUIRED_KEYS = createdTables.flatMap(({ table, lines }) =>
lines.flatMap((line) => {
const inline = /^(\w+) .*\bPRIMARY KEY\b/.exec(line);
const columns = inline?.[1] ?? keyColumns(line, "PRIMARY KEY");
if (columns) return [{ table, columns, primary: true }];
const unique = keyColumns(line, "UNIQUE");
return unique ? [{ table, columns: unique, primary: false }] : [];
})
);

View file

@ -1,7 +1,17 @@
import { describe, expect, it } from "vitest";
import { randomUUID } from "node:crypto";
import { Pool } from "pg";
import { afterEach, describe, expect, it } from "vitest";
import { newDb } from "pg-mem";
import { type HeimdallConfig } from "../src/config.js";
import { createStore } from "../src/store/index.js";
import { createPostgresStore, PostgresStore } from "../src/store/postgres.js";
import { REQUIRED_COLUMNS, REQUIRED_RELATIONS } from "../src/store/schema.js";
import {
REQUIRED_COLUMNS,
REQUIRED_KEYS,
REQUIRED_RELATIONS,
REQUIRED_TABLE_PRIVILEGES,
REQUIRED_TABLES,
} from "../src/store/schema.js";
describe("PostgresStore", () => {
it("round-trips core auth records through postgres", async () => {
@ -212,33 +222,155 @@ describe("createPostgresStore", () => {
});
describe("PostgresStore.checkSchema", () => {
it("requires every table and index the schema creates and every column it adds", () => {
it("requires every table, index, column and key the schema makes", () => {
expect(REQUIRED_RELATIONS).toEqual(
expect.arrayContaining(["accounts", "audit_events", "auth_completions_attempt_unconsumed_unique_idx", "audit_events_lookup_idx"])
);
expect(REQUIRED_RELATIONS).toHaveLength(17);
expect(REQUIRED_COLUMNS).toEqual([{ table: "auth_completions", column: "attempt_id" }]);
expect(REQUIRED_TABLES).toEqual([
"accounts",
"linked_identities",
"sessions",
"auth_attempts",
"private_command_receipts",
"auth_completions",
"capability_grants",
"entitlement_snapshots",
"audit_events",
]);
expect(REQUIRED_COLUMNS).toHaveLength(85);
expect(REQUIRED_COLUMNS).toEqual(
expect.arrayContaining([
{ table: "auth_completions", column: "code" },
{ table: "auth_completions", column: "mode" },
{ table: "linked_identities", column: "profile_json" },
{ table: "auth_completions", column: "attempt_id" },
])
);
// The keys the store's ON CONFLICT clauses and the foreign keys rely on.
expect(REQUIRED_KEYS).toEqual(
expect.arrayContaining([
{ table: "linked_identities", columns: "provider,provider_user_id", primary: false },
{ table: "sessions", columns: "id", primary: true },
{ table: "private_command_receipts", columns: "app_slug,idempotency_key", primary: true },
{ table: "entitlement_snapshots", columns: "account_id,provider,scope", primary: false },
{ table: "accounts", columns: "id", primary: true },
{ table: "auth_completions", columns: "code", primary: true },
])
);
expect(REQUIRED_KEYS).toHaveLength(11);
expect(REQUIRED_TABLE_PRIVILEGES).toEqual(["SELECT", "INSERT", "UPDATE"]);
});
function storeAnswering(missing: string[]) {
const queries: unknown[][] = [];
const store = new PostgresStore({
query: (async (_text: string, values: unknown[]) => {
queries.push(values);
return { rows: missing.map((name) => ({ missing: name })) };
}) as never,
function storeAnswering(kinds: string[]) {
return new PostgresStore({
query: (async () => ({ rows: kinds.map((kind) => ({ kind })) })) as never,
end: async () => undefined,
});
return { store, queries };
}
it("refuses with SCHEMA_MISSING when anything is missing", async () => {
const { store, queries } = storeAnswering(["auth_completions.attempt_id"]);
await expect(store.checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" });
expect(queries).toEqual([[REQUIRED_RELATIONS, ["auth_completions"], ["attempt_id"]]]);
it("refuses with SCHEMA_MISSING when anything is missing, whatever else is", async () => {
await expect(storeAnswering(["privilege", "missing"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_MISSING" });
});
it("refuses with SCHEMA_PRIVILEGES when only a privilege is missing", async () => {
await expect(storeAnswering(["privilege"]).checkSchema()).rejects.toMatchObject({ code: "SCHEMA_PRIVILEGES" });
});
it("passes when nothing is missing", async () => {
await expect(storeAnswering([]).store.checkSchema()).resolves.toBeUndefined();
await expect(storeAnswering([]).checkSchema()).resolves.toBeUndefined();
});
});
// Against a real Postgres, through the startup path itself: createStore, with
// and without applying the schema. HEIMDALL_TEST_PG_ADMIN_URL names a role
// that may create databases and roles; each case gets its own database.
const adminUrl = process.env.HEIMDALL_TEST_PG_ADMIN_URL;
describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_ADMIN_URL)", () => {
const refusal = (code: string) => `Postgres storage could not be prepared (${code}); check GC_ACCESS_DATABASE_URL_FILE.`;
const cleanup: Array<() => Promise<void>> = [];
afterEach(async () => {
for (const step of cleanup.splice(0).reverse()) await step();
});
function urlFor(database: string, user?: string, password?: string): string {
const url = new URL(adminUrl!);
url.pathname = `/${database}`;
if (user) url.username = user;
if (password) url.password = password;
return url.toString();
}
/** A fresh database with the schema applied by its owner, and an owner pool on it. */
async function appliedDatabase(): Promise<{ name: string; pool: Pool }> {
const name = `heimdall_check_${randomUUID().replace(/-/g, "")}`;
const admin = new Pool({ connectionString: adminUrl });
await admin.query(`CREATE DATABASE ${name}`);
cleanup.push(async () => {
await admin.query(`DROP DATABASE IF EXISTS ${name} WITH (FORCE)`);
await admin.end();
});
const pool = new Pool({ connectionString: urlFor(name) });
cleanup.push(async () => {
await pool.end();
});
await new PostgresStore(pool).ensureSchema();
return { name, pool };
}
async function start(url: string, applySchemaOnStartup: boolean): Promise<string> {
try {
const store = await createStore({ storage: { backend: "postgres", databaseUrl: url, applySchemaOnStartup } } as HeimdallConfig);
await store.close();
return "started";
} catch (error) {
return (error as Error).message;
}
}
it("starts on an applied database, applying or not", async () => {
const { name } = await appliedDatabase();
expect(await start(urlFor(name), false)).toBe("started");
expect(await start(urlFor(name), true)).toBe("started");
});
it("refuses a role that may only read, by code and without its password", async () => {
const { name, pool } = await appliedDatabase();
const role = `heimdall_ro_${randomUUID().replace(/-/g, "").slice(0, 12)}`;
await pool.query(`CREATE ROLE ${role} LOGIN PASSWORD 'CANARYrolepw'`);
cleanup.push(async () => {
await pool.query(`REVOKE ALL ON ALL TABLES IN SCHEMA public FROM ${role}`);
await pool.query(`DROP ROLE IF EXISTS ${role}`);
});
await pool.query(`GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${role}`);
const message = await start(urlFor(name, role, "CANARYrolepw"), false);
expect(message).toBe(refusal("SCHEMA_PRIVILEGES"));
expect(message).not.toContain("CANARY");
});
it("refuses a table whose primary key was dropped, even after applying the schema", async () => {
const { name, pool } = await appliedDatabase();
await pool.query("ALTER TABLE auth_completions DROP CONSTRAINT auth_completions_pkey");
expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING"));
expect(await start(urlFor(name), true)).toBe(refusal("SCHEMA_MISSING"));
});
it("refuses a unique key the store's ON CONFLICT needs, replaced by a partial one", async () => {
const { name, pool } = await appliedDatabase();
await pool.query("ALTER TABLE entitlement_snapshots DROP CONSTRAINT entitlement_snapshots_account_id_provider_scope_key");
await pool.query(
"CREATE UNIQUE INDEX entitlement_snapshots_partial ON entitlement_snapshots(account_id, provider, scope) WHERE is_allowed"
);
expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING"));
});
it("refuses a created table's dropped column", async () => {
const { name, pool } = await appliedDatabase();
await pool.query("ALTER TABLE auth_completions DROP COLUMN mode");
expect(await start(urlFor(name), false)).toBe(refusal("SCHEMA_MISSING"));
});
});