diff --git a/src/store/index.ts b/src/store/index.ts index 972bd8f..69abbe6 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -32,9 +32,13 @@ export async function createStore(config: HeimdallConfig): Promise undefined); - throw new Error( - `Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); check GC_ACCESS_DATABASE_URL_FILE.` - ); + // A role that may not create temporary tables is not a bad URL, so that + // refusal names the privilege instead. + const remedy = + code === "SCHEMA_TEMP_PRIVILEGE" + ? "grant the database role TEMPORARY on its database" + : "check GC_ACCESS_DATABASE_URL_FILE"; + throw new Error(`Postgres storage could not be prepared (${typeof code === "string" ? code : "error"}); ${remedy}.`); } return store; diff --git a/src/store/postgres.ts b/src/store/postgres.ts index e589f2e..64f013d 100644 --- a/src/store/postgres.ts +++ b/src/store/postgres.ts @@ -363,36 +363,6 @@ export function samePredicate(declared: string, actual: string, keyColumns: stri return left.length === right.length && left.every((condition, index) => condition === right[index]); } -/** - * Each REQUIRED_KEYS predicate as Postgres prints it back (empty for none). - * One simple-protocol round trip on one connection: a transaction that - * builds a temporary copy of each keyed table's declared columns, indexes - * it with the declared predicate, reads the printed form and rolls back. - * The names and SQL are the schema's own constants, never input. - */ -export async function printedDeclaredPredicates(pool: Pick): Promise { - const partial = REQUIRED_KEYS.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); - if (partial.length === 0) return REQUIRED_KEYS.map(() => ""); - const statements = partial.flatMap(({ table, columns, predicate, index }) => { - const probe = `heimdall_key_probe_${index}`; - const declared = REQUIRED_COLUMNS.filter((column) => column.table === table) - .map((column) => `${column.column} ${column.type}`) - .join(", "); - return [ - `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, - `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, - ]; - }); - const select = partial - .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) - .join(" UNION ALL "); - const results = (await pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as Array<{ - rows: Array<{ key: number; predicate: string }>; - }>; - const printed = results[results.length - 2]!.rows; - return REQUIRED_KEYS.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); -} - /** * What the store's statements do to each table, and so the privileges its * database role needs: it reads what it returns or looks up, inserts what it @@ -411,6 +381,170 @@ export const STORE_TABLE_PRIVILEGES: Readonly>; +} + +/** What Heimdall's store needs, read from its schema and its statements. */ +export const HEIMDALL_SCHEMA_REQUIREMENTS: SchemaRequirements = { + tables: REQUIRED_TABLES, + columns: REQUIRED_COLUMNS, + keys: REQUIRED_KEYS, + privileges: STORE_TABLE_PRIVILEGES, +}; + +/** + * Each required key's predicate as Postgres prints it back (empty for none). + * One simple-protocol round trip on one connection: a transaction that builds + * a temporary copy of each keyed table's declared columns, indexes it with the + * declared predicate, reads the printed form and rolls back. The names and SQL + * come from the requirements, which are constants, never input. A role that + * may not create temporary tables fails with code SCHEMA_TEMP_PRIVILEGE. + */ +export async function printedDeclaredPredicates(pool: Pick, requirements: SchemaRequirements): Promise { + const partial = requirements.keys.flatMap((required, index) => (required.predicate ? [{ ...required, index }] : [])); + if (partial.length === 0) return requirements.keys.map(() => ""); + const statements = partial.flatMap(({ table, columns, predicate, index }) => { + const probe = `heimdall_key_probe_${index}`; + const declared = requirements.columns + .filter((column) => column.table === table) + .map((column) => `${column.column} ${column.type}`) + .join(", "); + return [ + `CREATE TEMP TABLE ${probe} (${declared}) ON COMMIT DROP`, + `CREATE UNIQUE INDEX ${probe}_idx ON ${probe} (${columns}) WHERE ${predicate}`, + ]; + }); + const select = partial + .map(({ index }) => `SELECT ${index} AS key, pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'pg_temp.heimdall_key_probe_${index}_idx'::regclass`) + .join(" UNION ALL "); + let results: Array<{ rows: Array<{ key: number; predicate: string }> }>; + try { + results = (await pool.query(["BEGIN", ...statements, select, "ROLLBACK"].join(";\n"))) as unknown as typeof results; + } catch (error) { + if ((error as { code?: unknown }).code === "42501") { + throw Object.assign(new Error("The Heimdall database role may not create temporary tables."), { code: "SCHEMA_TEMP_PRIVILEGE" }); + } + throw error; + } + const printed = results[results.length - 2]!.rows; + return requirements.keys.map((_, index) => printed.find((row) => row.key === index)?.predicate ?? ""); +} + +/** + * Proves that the database can take a store's requests: every required + * table; every column with its declared type, under a deterministic + * collation, and nullable where declared nullable; every unique key; and + * every privilege, held on the table or on each of its columns. A missing or + * altered piece fails with code SCHEMA_MISSING and a missing privilege with + * SCHEMA_PRIVILEGES; neither error names the piece. + * + * A key is met by a unique index on its table that is valid (a failed + * CREATE INDEX CONCURRENTLY or an interrupted DROP INDEX CONCURRENTLY leaves + * one that is not; Postgres never marks an index valid before it is ready), + * immediate (ON CONFLICT refuses a deferrable arbiter), has exactly the key's + * columns as its key columns (INCLUDE columns aside), compares them under + * deterministic collations (a case-insensitive one would merge two + * providers' user ids into one row), and has the key's predicate. A + * predicate matches when both, as Postgres prints them, are the same set of + * AND-ed conditions; a `key_column IS NOT NULL` condition may be absent on + * either side, because a unique index never makes NULL keys collide unless + * it is NULLS NOT DISTINCT. Postgres prints the declared predicates itself + * (printedDeclaredPredicates). + * + * What this does not prove is listed where the schema is: see schema.ts. + */ +export async function checkSchemaRequirements(pool: Pick, requirements: SchemaRequirements): Promise { + const declaredPredicates = await printedDeclaredPredicates(pool, requirements); + const privileges = Object.entries(requirements.privileges).flatMap(([table, wanted]) => + wanted.map((privilege) => ({ table, privilege })) + ); + const result = await pool.query<{ + kind: "missing" | "privilege" | "candidate"; + key: number | null; + predicate: string | null; + nulls_not_distinct: boolean | null; + }>( + ` + SELECT 'missing' AS kind, NULL::int AS key, NULL::text AS predicate, NULL::boolean AS nulls_not_distinct + FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL + UNION ALL + SELECT 'missing', NULL, NULL, NULL + FROM unnest($2::text[], $3::text[], $4::text[], $5::boolean[]) AS required(table_name, column_name, type_name, nullable) + WHERE NOT EXISTS ( + SELECT 1 FROM pg_attribute a + WHERE a.attrelid = to_regclass(required.table_name) + AND a.attname = required.column_name + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.atttypid = to_regtype(required.type_name) + AND (NOT required.nullable OR NOT a.attnotnull) + AND NOT EXISTS (SELECT 1 FROM pg_collation c WHERE c.oid = a.attcollation AND NOT c.collisdeterministic) + ) + UNION ALL + SELECT 'candidate', required.key::int, pg_get_expr(i.indpred, i.indrelid), + coalesce((to_jsonb(i) ->> 'indnullsnotdistinct')::boolean, false) + FROM unnest($6::text[], $7::text[]) WITH ORDINALITY AS required(table_name, columns, key) + JOIN pg_index i ON i.indrelid = to_regclass(required.table_name) + WHERE i.indisunique + AND i.indisvalid + AND i.indimmediate + AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) + AND ( + SELECT string_agg(a.attname, ',' ORDER BY a.attname) + FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum + WHERE key.position <= i.indnkeyatts + ) = required.columns + AND NOT EXISTS ( + SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS key_collation(oid, position) + JOIN pg_collation c ON c.oid = key_collation.oid + WHERE key_collation.position <= i.indnkeyatts AND NOT c.collisdeterministic + ) + UNION ALL + SELECT 'privilege', NULL, NULL, NULL + FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) + WHERE EXISTS ( + SELECT 1 FROM unnest($2::text[], $3::text[]) AS used(table_name, column_name) + JOIN pg_attribute a ON a.attrelid = to_regclass(used.table_name) AND a.attname = used.column_name AND NOT a.attisdropped + WHERE used.table_name = required.table_name + AND NOT has_column_privilege(a.attrelid, a.attnum, required.privilege) + ) + `, + [ + requirements.tables, + requirements.columns.map((required) => required.table), + requirements.columns.map((required) => required.column), + requirements.columns.map((required) => required.type), + requirements.columns.map((required) => required.nullable), + requirements.keys.map((required) => required.table), + requirements.keys.map((required) => required.columns), + privileges.map((required) => required.table), + privileges.map((required) => required.privilege), + ] + ); + const keyMet = requirements.keys.map((required, index) => + result.rows.some( + (row) => + row.kind === "candidate" && + row.key === index + 1 && + samePredicate(declaredPredicates[index]!, row.predicate ?? "", required.columns.split(","), row.nulls_not_distinct === true) + ) + ); + if (result.rows.some((row) => row.kind === "missing") || keyMet.includes(false)) { + throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); + } + if (result.rows.some((row) => row.kind === "privilege")) { + throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { + code: "SCHEMA_PRIVILEGES", + }); + } +} + export class PostgresStore implements HeimdallStore { constructor(private readonly pool: Pick) {} @@ -418,113 +552,9 @@ export class PostgresStore implements HeimdallStore { await this.pool.query(CREATE_SCHEMA_SQL); } - /** - * Proves, before Heimdall serves, that the database can take its requests: - * every table the schema makes; every column with its declared type, and - * nullable where the schema leaves it nullable; every unique key; and every - * privilege in STORE_TABLE_PRIVILEGES, held on the table or on each of its - * columns. A missing or altered piece fails with code SCHEMA_MISSING and a - * missing privilege with SCHEMA_PRIVILEGES; neither error names the piece. - * - * A key is met by a unique index on its table that is valid (a failed - * CREATE INDEX CONCURRENTLY or an interrupted DROP INDEX CONCURRENTLY leaves - * one that is not; Postgres never marks an index valid before it is ready), - * immediate (ON CONFLICT refuses a deferrable arbiter), has exactly the key's columns - * as its key columns (INCLUDE columns aside), compares them under - * deterministic collations (a case-insensitive one would merge two - * providers' user ids into one row), and has the key's predicate. A - * predicate matches when both, as Postgres prints them, are the same set of - * AND-ed conditions; a `key_column IS NOT NULL` condition may be absent on - * either side, because a unique index never makes NULL keys collide unless - * it is NULLS NOT DISTINCT. - * - * Postgres prints the declared predicates itself: they are indexed on a - * temporary copy of the table in a transaction that is rolled back. - */ + /** Proves, before Heimdall serves, that the database can take its requests (checkSchemaRequirements). */ async checkSchema(): Promise { - const declaredPredicates = await printedDeclaredPredicates(this.pool); - const privileges = Object.entries(STORE_TABLE_PRIVILEGES).flatMap(([table, wanted]) => - wanted.map((privilege) => ({ table, privilege })) - ); - const result = await this.pool.query<{ - kind: "missing" | "privilege" | "candidate"; - key: number | null; - predicate: string | null; - nulls_not_distinct: boolean | null; - }>( - ` - SELECT 'missing' AS kind, NULL::int AS key, NULL::text AS predicate, NULL::boolean AS nulls_not_distinct - FROM unnest($1::text[]) AS name WHERE to_regclass(name) IS NULL - UNION ALL - SELECT 'missing', NULL, NULL, NULL - FROM unnest($2::text[], $3::text[], $4::text[], $5::boolean[]) AS required(table_name, column_name, type_name, nullable) - WHERE NOT EXISTS ( - SELECT 1 FROM pg_attribute a - WHERE a.attrelid = to_regclass(required.table_name) - AND a.attname = required.column_name - AND a.attnum > 0 - AND NOT a.attisdropped - AND a.atttypid = to_regtype(required.type_name) - AND (NOT required.nullable OR NOT a.attnotnull) - ) - UNION ALL - SELECT 'candidate', required.key::int, pg_get_expr(i.indpred, i.indrelid), - coalesce((to_jsonb(i) ->> 'indnullsnotdistinct')::boolean, false) - FROM unnest($6::text[], $7::text[]) WITH ORDINALITY AS required(table_name, columns, key) - JOIN pg_index i ON i.indrelid = to_regclass(required.table_name) - WHERE i.indisunique - AND i.indisvalid - AND i.indimmediate - AND i.indnkeyatts = cardinality(string_to_array(required.columns, ',')) - AND ( - SELECT string_agg(a.attname, ',' ORDER BY a.attname) - FROM unnest(i.indkey::int2[]) WITH ORDINALITY AS key(attnum, position) - JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = key.attnum - WHERE key.position <= i.indnkeyatts - ) = required.columns - AND NOT EXISTS ( - SELECT 1 FROM unnest(i.indcollation::oid[]) WITH ORDINALITY AS key_collation(oid, position) - JOIN pg_collation c ON c.oid = key_collation.oid - WHERE key_collation.position <= i.indnkeyatts AND NOT c.collisdeterministic - ) - UNION ALL - SELECT 'privilege', NULL, NULL, NULL - FROM unnest($8::text[], $9::text[]) AS required(table_name, privilege) - WHERE EXISTS ( - SELECT 1 FROM unnest($2::text[], $3::text[]) AS used(table_name, column_name) - JOIN pg_attribute a ON a.attrelid = to_regclass(used.table_name) AND a.attname = used.column_name AND NOT a.attisdropped - WHERE used.table_name = required.table_name - AND NOT has_column_privilege(a.attrelid, a.attnum, required.privilege) - ) - `, - [ - REQUIRED_TABLES, - REQUIRED_COLUMNS.map((required) => required.table), - REQUIRED_COLUMNS.map((required) => required.column), - REQUIRED_COLUMNS.map((required) => required.type), - REQUIRED_COLUMNS.map((required) => required.nullable), - REQUIRED_KEYS.map((required) => required.table), - REQUIRED_KEYS.map((required) => required.columns), - privileges.map((required) => required.table), - privileges.map((required) => required.privilege), - ] - ); - const keyMet = REQUIRED_KEYS.map((required, index) => - result.rows.some( - (row) => - row.kind === "candidate" && - row.key === index + 1 && - samePredicate(declaredPredicates[index]!, row.predicate ?? "", required.columns.split(","), row.nulls_not_distinct === true) - ) - ); - if (result.rows.some((row) => row.kind === "missing") || keyMet.includes(false)) { - throw Object.assign(new Error("The Heimdall schema is not applied."), { code: "SCHEMA_MISSING" }); - } - if (result.rows.some((row) => row.kind === "privilege")) { - throw Object.assign(new Error("The Heimdall database role lacks a privilege the store needs."), { - code: "SCHEMA_PRIVILEGES", - }); - } + await checkSchemaRequirements(this.pool, HEIMDALL_SCHEMA_REQUIREMENTS); } async close(): Promise { diff --git a/tests/store.test.ts b/tests/store.test.ts index a94f76f..a7a45f5 100644 --- a/tests/store.test.ts +++ b/tests/store.test.ts @@ -7,6 +7,8 @@ import { createStore } from "../src/store/index.js"; import { createPostgresStore, PostgresStore, + checkSchemaRequirements, + HEIMDALL_SCHEMA_REQUIREMENTS, predicateConditions, printedDeclaredPredicates, samePredicate, @@ -519,7 +521,7 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD it("prints each declared predicate exactly as Postgres prints the applied index's", async () => { const { owner } = await appliedDatabase(); - const printed = await printedDeclaredPredicates(owner); + const printed = await printedDeclaredPredicates(owner, HEIMDALL_SCHEMA_REQUIREMENTS); const applied = await owner.query<{ predicate: string }>( "SELECT pg_get_expr(indpred, indrelid) AS predicate FROM pg_index WHERE indexrelid = 'auth_completions_attempt_unconsumed_unique_idx'::regclass" ); @@ -552,6 +554,51 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD await exerciseStore(new PostgresStore(pool)); }); + it("refuses a privilege missing on one column of another table", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, { ...STORE_TABLE_PRIVILEGES, auth_completions: [] }); + const columns = REQUIRED_COLUMNS.filter((column) => column.table === "auth_completions" && column.column !== "attempt_id") + .map((column) => column.column) + .join(", "); + for (const privilege of STORE_TABLE_PRIVILEGES.auth_completions!) { + await owner.query(`GRANT ${privilege} (${columns}) ON auth_completions TO ${role}`); + } + + expect(await start(urlFor(name, role, rolePassword), false)).toBe(refusal("SCHEMA_PRIVILEGES")); + }); + + it("names the TEMPORARY privilege when the role may not create temporary tables", async () => { + const { name, owner } = await appliedDatabase(); + await grant(owner, STORE_TABLE_PRIVILEGES); + await owner.query(`REVOKE TEMPORARY ON DATABASE ${name} FROM PUBLIC`); + + expect(await start(urlFor(name, role, rolePassword), false)).toBe( + "Postgres storage could not be prepared (SCHEMA_TEMP_PRIVILEGE); grant the database role TEMPORARY on its database." + ); + }); + + // The schema's own predicate splits into the same conditions raw or + // printed, so only a predicate whose printed form differs (a literal gains + // its cast) shows that the check compares Postgres's printing. + it("matches a declared predicate that Postgres prints differently from how it is written", async () => { + const { owner } = await appliedDatabase(); + await owner.query("CREATE TABLE predicate_probe (id INTEGER NOT NULL, state TEXT, attempts INTEGER)"); + await owner.query("CREATE UNIQUE INDEX predicate_probe_live ON predicate_probe (id) WHERE state = 'live' AND attempts > 0"); + const requirements = { + tables: ["predicate_probe"], + columns: [ + { table: "predicate_probe", column: "id", type: "INTEGER", nullable: false }, + { table: "predicate_probe", column: "state", type: "TEXT", nullable: true }, + { table: "predicate_probe", column: "attempts", type: "INTEGER", nullable: true }, + ], + keys: [{ table: "predicate_probe", columns: "id", predicate: "state = 'live' AND attempts > 0" }], + privileges: {}, + }; + + expect((await printedDeclaredPredicates(owner, requirements))[0]).toBe("((state = 'live'::text) AND (attempts > 0))"); + await expect(checkSchemaRequirements(owner, requirements)).resolves.toBeUndefined(); + }); + it("refuses a privilege held on only some of a table's columns", async () => { const { name, owner } = await appliedDatabase(); await grant(owner, { ...STORE_TABLE_PRIVILEGES, audit_events: [] }); @@ -598,6 +645,18 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD ["ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", "CREATE UNIQUE INDEX li_half_dropped ON linked_identities(provider, provider_user_id)", "UPDATE pg_index SET indisvalid = false WHERE indexrelid = 'li_half_dropped'::regclass"]], + ["a key's first column compared under a case-insensitive collation", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "CREATE UNIQUE INDEX li_ci_first ON linked_identities(provider COLLATE heimdall_ci, provider_user_id)"]], + // Soul pass 7: the key index compares under "C", but the column itself is + // case-insensitive, so the store's own lookups by provider_user_id match + // other users' ids. + ["a column declared under a case-insensitive collation, its key index under C", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "ALTER TABLE linked_identities ALTER COLUMN provider_user_id TYPE text COLLATE heimdall_ci", + 'CREATE UNIQUE INDEX li_c ON linked_identities(provider, provider_user_id COLLATE "C")']], ["a key compared under a case-insensitive collation (two users' ids become one row)", ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", @@ -617,6 +676,8 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD ["a column retyped", ["ALTER TABLE sessions ALTER COLUMN access_revision TYPE TEXT"]], ["a created table's column dropped", ["ALTER TABLE auth_completions DROP COLUMN mode"]], ["NOT NULL added to a column the store leaves null", ["ALTER TABLE auth_completions ALTER COLUMN attempt_id SET NOT NULL"]], + ["NOT NULL added to a nullable column of another table", + ["UPDATE linked_identities SET username = ''", "ALTER TABLE linked_identities ALTER COLUMN username SET NOT NULL"]], ])("refuses %s", async (_damage, steps) => { const { name, owner } = await appliedDatabase(); for (const step of steps) { @@ -639,6 +700,13 @@ describe.skipIf(!adminUrl)("startup against a live Postgres (HEIMDALL_TEST_PG_AD ["the attempt index without its redundant attempt_id IS NOT NULL", ["DROP INDEX auth_completions_attempt_unconsumed_unique_idx", "CREATE UNIQUE INDEX attempt_unconsumed ON auth_completions(app_slug, attempt_id) WHERE consumed_at IS NULL"]], ["a lookup index dropped", ["DROP INDEX audit_events_lookup_idx", "DROP INDEX auth_completions_lookup_idx"]], + // Only a key's own columns decide which rows collide; an INCLUDEd column's + // collation does not, whatever it is. + ["a key index that INCLUDEs an undeclared case-insensitive column", + ["CREATE COLLATION heimdall_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)", + "ALTER TABLE linked_identities ADD COLUMN operator_note text COLLATE heimdall_ci", + "ALTER TABLE linked_identities DROP CONSTRAINT linked_identities_provider_provider_user_id_key", + "CREATE UNIQUE INDEX li_note ON linked_identities(provider, provider_user_id) INCLUDE (operator_note)"]], ])("accepts %s", async (_change, steps) => { const { name, owner } = await appliedDatabase(); for (const step of steps) await owner.query(step);