diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 53eb5867..8328cb86 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,7 +20,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. -- A record whose payload carries a slot past its reader's declared slots, at any depth (the record, a nested record, a list element, a union arm, a unit arm), is now held read-only: it loads and reads, and every `put`, `putEnvelope`, `update` or `delete` of its key is refused with `SchemaConflictError` before a store is touched (cultlib-gaps:ruling:extra-trailing-slots, as the C# reference). A load used to skip the extra slots silently and a later write shed them. The refusal names the schema, the key, the type that was read with extra slots and both slot counts, never a slot value, and carries them as `SchemaConflictError.unreadSlots`. An untouched record keeps its bytes through writes of other keys. Missing trailing slots keep their declared defaults and mark nothing. A record put through the cache never carries a mark. A mark at depth 0 is consented to only when the record arrived under a catalog entry that lists those slots and the reader lists that id compatible (`unreadSlots.explained`); every other mark refuses whatever is declared. New: `CultCache.isWritable(definition, key)` (live, advisory), `CultCache.unreadSlots(definition, key)` and the exported `CultCacheUnreadSlots`. Documents with no slot codec (a plain map, a user formatter) never mark. Implied semver: a minor bump (0.16.0) while the package is 0.x; a major one at 1.0 and after. +- A record whose payload carries a slot past its reader's declared slots, at any depth (the record, a nested record, a list element, a union arm, a unit arm), is now held read-only: it loads and reads, and every `put`, `putEnvelope`, `update` or `delete` of its key is refused with `SchemaConflictError` before a store is touched (cultlib-gaps:ruling:extra-trailing-slots, as the C# reference). A load used to skip the extra slots silently and a later write shed them. The refusal names the schema, the key, the type that was read with extra slots and both slot counts, never a slot value, and carries them as `SchemaConflictError.unreadSlots`. An untouched record keeps its bytes through writes of other keys. Missing trailing slots keep their declared defaults and mark nothing. A `put` never marks (the value is encoded by this runtime); a `putEnvelope` of a payload carrying extra slots lands its bytes and holds the record read-only. A mark at depth 0 is consented to only when the record arrived under a schema id other than the reader's own that the reader lists compatible, and that id's catalog entry lists those slots (a writer that adds a slot and keeps the reader's id is never consented to) (`unreadSlots.explained`); every other mark refuses whatever is declared. New: `CultCache.isWritable(definition, key)` (live, advisory), `CultCache.unreadSlots(definition, key)` and the exported `CultCacheUnreadSlots`. Documents with no slot codec (a plain map, a user formatter) never mark. Implied semver: a minor bump (0.16.0) while the package is 0.x; a major one at 1.0 and after. ### Added diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index e370e2e8..3f15d378 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -54,6 +54,8 @@ type HydratedEntry = CultCacheEnvelope & { lookups: EntryLookups; // What the payload decode read past: set when the record was admitted, and it makes the record read-only unless explained. unreadSlots?: CultCacheUnreadSlots; + // The schema id the record arrived under, before it was admitted under its reader's: what an unread-slots refusal names. + storedSchemaId?: string; }; // Everything a pull replaces. Built whole, then swapped in with one assignment. @@ -88,11 +90,15 @@ type DerivedTypeLookups = { readonly indexMaps: Map>; }; -// The consent a compatible-id declaration gives (cultlib-gaps:ruling:slot-mark-consent): a mark is explained only when it is at +// The consent a compatible-id declaration gives (cultlib-gaps:ruling:slot-mark-consent): a mark is explained only when the record +// arrived under an id other than the reader's own that the reader lists compatible, it is at // depth 0 and the schema comparison of the catalog entry the record arrived with against the reader's own reported the slots it // carries as ignored, so its slot count is at most the highest ignored slot plus one. A record written or put through the cache // arrives under the reader's own entry, which ignores nothing, so no mark of it is explained. -function explainedBy(noted: NotedSlots, reader: CultCacheSchemaCatalogEntry, arrived: CultCacheSchemaCatalogEntry | undefined): boolean { +function explainedBy(noted: NotedSlots, reader: CultCacheSchemaCatalogEntry, arrivedId: string, arrived: CultCacheSchemaCatalogEntry | undefined): boolean { + if (arrivedId === reader.schemaId || !(reader.compatibleSchemaIds ?? []).includes(arrivedId)) { + return false; + } const declared = new Set((reader.members ?? []).map((member) => member.slot)); const ignored = (arrived?.members ?? []).map((member) => member.slot).filter((slot) => !declared.has(slot)); return noted.depth0Only && noted.slotCount <= Math.max(-1, ...ignored) + 1; @@ -599,7 +605,6 @@ export class CultCache { }); } - // What the last pull found that no registered type claims, by key ordinal. A later registration claims a record at the next pull. /** What the record's payload carried past this build's declared slots, or undefined: the held record is read-only unless `explained`. */ unreadSlots(definition: AnyCultCacheDocumentDefinition, key: string): CultCacheUnreadSlots | undefined { return this.#hydrated.entries.get(this.#entryId(this.#requireDefinition(definition).definition.type, key))?.unreadSlots; @@ -612,10 +617,21 @@ export class CultCache { */ isWritable(definition: AnyCultCacheDocumentDefinition, key: string): boolean { const registered = this.#requireDefinition(definition); - return !this.#hydrated.foreign.some((record) => record.key === key) - && this.#unreadRefusal(this.#hydrated.entries.get(this.#entryId(registered.definition.type, key))) === undefined; + const type = registered.definition.type; + if (this.#hydrated.foreign.some((record) => record.key === key) + || this.#unreadRefusal(this.#hydrated.entries.get(this.#entryId(type, key))) !== undefined) { + return false; + } + // The record of another type a write would restamp (or be refused by): the write's own questions, writeNow's #refuseRetype. + const other = this.#heldUnderOtherType(this.#hydrated, type, key); + const written: CultCacheEnvelope = { + key, type, payload: documentPayloadFromBytes(new Uint8Array(0)), storedAt: "", + schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, + }; + return other === undefined || (writtenOwns(written, schemaIdForEnvelope(other)) && this.#unreadRefusal(other) === undefined); } + // What the last pull found that no registered type claims, by key ordinal. A later registration claims a record at the next pull. foreignRecords(): readonly CultCacheForeignRecord[] { return [...this.#hydrated.foreign].sort((left, right) => compareOrdinal(left.key, right.key)); } @@ -765,7 +781,7 @@ export class CultCache { return undefined; } return new SchemaConflictError( - held.schemaId ?? held.type, + held.storedSchemaId ?? held.schemaId ?? held.type, [held.catalogEntry.schemaName], held.key, `${held.catalogEntry.schemaName} record ${held.key} is read-only: ${unread.typeName} was read with ${unread.slotCount} slots where this build declares ${unread.declaredCount} ` + @@ -1078,7 +1094,8 @@ export class CultCache { catalogEntry: registered.catalogEntry, value, lookups, - ...(noted ? { unreadSlots: { ...noted, explained: explainedBy(noted, registered.catalogEntry, entry.catalogEntry) } } : {}), + storedSchemaId: entry.schemaId ?? entry.type, + ...(noted ? { unreadSlots: { ...noted, explained: explainedBy(noted, registered.catalogEntry, entry.schemaId ?? entry.type, entry.catalogEntry) } } : {}), }; } diff --git a/packages/cultcache-ts/test/slot-readonly.test.ts b/packages/cultcache-ts/test/slot-readonly.test.ts index 262362b0..364adc67 100644 --- a/packages/cultcache-ts/test/slot-readonly.test.ts +++ b/packages/cultcache-ts/test/slot-readonly.test.ts @@ -5,7 +5,9 @@ import { join } from "node:path"; import { test } from "node:test"; import { decode, encode } from "@msgpack/msgpack"; +import { z } from "zod"; +import { codecs } from "../src"; import { CultCache } from "../src/cult-cache"; import { defineDocumentType, documentPayloadBytes } from "../src/document"; import { SingleFileMessagePackBackingStore, readSingleFileStore } from "../src/single-file-messagepack-backing-store"; @@ -19,6 +21,7 @@ import { // read-only, and every write or removal of its key is refused before a store is touched. The vectors are written by the C# // reference: tests/vectors/union-parity/README.md, "Extra trailing slot" and "Slot stores". +const { slots, string } = codecs; const SLOT_VALUE = "extra-slot"; function mark(typeName: string, slotCount: number, declaredCount: number, depth0Only: boolean, explained = false, occurrences = 1): CultCacheUnreadSlots { @@ -129,6 +132,7 @@ test("a write by a type that owns the held record's id restamps it, and is refus const held = { ...envelopeOf(await read(file)), type: older.type, schemaId: "tests.surface_older.v1" }; const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(memoryStore([held])).build(); await cache.pullAllBackingStores(); + assert.equal(cache.isWritable(newer, "surface"), lands, `${file}: isWritable agrees with the write by the type that owns the id`); const write = cache.put(newer, "surface", golden); if (lands) { await write; @@ -225,7 +229,11 @@ test("consent: a write of the explained record lands; the over-consent write is for (const key of ["over-consent", "nested"]) { const refused = await openStore("extra-slot-consent.cc", true); - await assert.rejects(() => refused.cache.put(refused.surface, key, golden), SchemaConflictError, key); + await assert.rejects( + () => refused.cache.put(refused.surface, key, golden), + (error: unknown) => error instanceof SchemaConflictError && error.schemaId === "tests.union_surface.wide", + `${key}: the refusal names the id the record was stored under`, + ); } }); @@ -243,3 +251,44 @@ test("carried: after writing another record, the marked record's bytes, schema i assert.deepEqual(now.catalogEntry, was.catalogEntry, key); } }); + +// A writer that adds a slot and keeps its schema id (TS ids are hand-written; C# ids hash the members): the reader of that +// same id has declared nothing, so it consents to nothing. +test("a wider writer under the reader's own schema id is read-only, and a real round trip does not shed the slot", async () => { + const doc = (columns: 1 | 2, compatibleSchemaIds: string[] = []) => defineDocumentType({ + type: "tests.own_id", + schemaId: "tests.own_id.v1", + schemaName: "tests.own_id", + compatibleSchemaIds, + schema: z.object({ n: z.string(), extra: z.string().optional() }), + codec: columns === 2 + ? slots<{ n: string; extra: string }>({ 0: ["n", string], 1: ["extra", string] }, "Own") + : slots<{ n: string }>({ 0: ["n", string] }, "Own"), + name: "n", + }); + const path = join(await mkdtemp(join(tmpdir(), "slot-readonly-")), "own.cc"); + const wide = doc(2); + const writer = CultCache.builder().withDocumentType(wide).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await writer.put(wide, "k", { n: "n1", extra: "kept" }); + const bytes = await readFile(path); + + for (const compatible of [[], ["tests.own_id.v1"]]) { + const reader = doc(1, compatible); + const cache = CultCache.builder().withDocumentType(reader).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await cache.pullAllBackingStores(); + assert.deepEqual(cache.unreadSlots(reader, "k"), mark("Own", 2, 1, true), "unexplained: the arrived id is the reader's own"); + assert.equal(cache.isWritable(reader, "k"), false); + await assert.rejects(() => cache.put(reader, "k", { n: "n2" }), SchemaConflictError); + await assert.rejects(() => cache.delete(reader, "k"), SchemaConflictError); + assert.equal(Buffer.compare(bytes, await readFile(path)), 0, "the file still holds the slot"); + } +}); + +test("a record that arrived under an id the reader does not list compatible is never consented to", async () => { + const path = await copyOf("extra-slot-consent.cc"); + const reader = surfaceDocument("tests.union_surface.v2"); // resolves by type name; lists nothing + const cache = CultCache.builder().withDocumentType(reader).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await cache.pullAllBackingStores(); + assert.equal(cache.unreadSlots(reader, "consented")?.explained, false); + assert.equal(cache.isWritable(reader, "consented"), false); +});