diff --git a/README.md b/README.md index b7509b2..c6b85be 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,16 @@ This is not an ORM in a fake mustache. It is for cases where you want: - document types are registered explicitly - a document schema only needs a `parse(...)` function, so generated JSON-schema contracts can plug in directly without growing a second hand-written Zod body -- each persisted envelope stores: - - `type` +- each persisted record stores: - `key` + - `schemaId` - raw MessagePack `payload` bytes - `storedAt` -- payload bytes only decode through the registered document definition for that `type` +- the single-file store writes the CultCache v1 snapshot shape: + - `formatVersion` + - embedded schema catalog + - record array +- payload bytes only decode through the registered document definition for that schema - name and index maps live in the cache, not in the backing store - global documents are treated as a singleton per type - backing stores are adapters, not the public data model @@ -41,6 +45,9 @@ import { const itemDocument = defineDocumentType({ type: "item", + schemaId: "gamecult.item.v1", + schemaName: "item", + schemaVersion: "item.v1", schema: z.object({ name: z.string(), category: z.string(), @@ -119,6 +126,10 @@ const settings = cache.getRequiredGlobal(settingsDocument); - `getGlobalEnvelope(...)` - `putEnvelope(...)` +Document definitions may also carry persistence metadata for cross-language +stores: `schemaId`, `schemaName`, `schemaVersion`, `contentHash`, +`canonicalSchemaJson`, `compatibleSchemaIds`, and slot `members`. + ## Name, Index, and Global Semantics `name` and `indexes` can be declared on the document definition: @@ -194,6 +205,10 @@ formatter contract, `CultCacheTS` can now move the persisted envelope directly: - `putEnvelope(...)` ingests that envelope into another cache instance without re-encoding the payload first +Raw envelopes may carry `schemaId` and catalog metadata. Domain callers should +not need that sludge. Replication, migration tooling, and wire-compatibility +tests do. + That still is not magic shared memory. The cache decodes once so lookups and typed reads stay honest. But it stops doing the stupid part where identical MessagePack payload bytes get decoded into a generic value, then re-encoded @@ -203,7 +218,7 @@ into the same bytes again just to cross the room. `CultCacheTS` should never deserialize arbitrary persisted payloads into arbitrary runtime shapes. -It only accepts persisted envelopes whose `type` discriminator resolves to a registered document definition on the cache instance. Unknown types fail immediately. Known types decode through the registered formatter and schema for that type only. +It only accepts persisted records whose `schemaId` or legacy `type` discriminator resolves to a registered document definition on the cache instance. Unknown schemas fail immediately. Known schemas decode through the registered formatter and schema only. That is the TypeScript version of the original CultCache trick: @@ -216,8 +231,12 @@ That is the TypeScript version of the original CultCache trick: `SingleFileMessagePackBackingStore` is the first concrete store. -- it writes atomic snapshots +- it writes atomic CultCache v1 MessagePack snapshots +- the top-level MessagePack value is `[formatVersion, schemaCatalog, records]` +- each record is `[key, schemaId, storedAt, payload]` +- each catalog entry is `[schemaId, schemaName, schemaVersion, contentHash, canonicalSchemaJson, compatibleSchemaIds, members]` +- it still reads the older TS/Rust-style envelope array and rewrites repaired legacy payloads as v1 snapshots - it queues local writes so one process does not step on its own shoelaces -- it persists raw MessagePack payload bytes inside MessagePack envelopes +- it persists raw MessagePack payload bytes inside schema-addressed records It is a control-plane store, not a tiny god. If multiple processes hammer the same file, you still need a coordinator or an external lock instead of prayer. diff --git a/dist/cult-cache.d.ts.map b/dist/cult-cache.d.ts.map index 8a61ac3..d3437a9 100644 --- a/dist/cult-cache.d.ts.map +++ b/dist/cult-cache.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"cult-cache.d.ts","sourceRoot":"","sources":["../src/cult-cache.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,8BAA8B,EAC9B,iBAAiB,EACjB,yBAAyB,EAGzB,yBAAyB,EACzB,sBAAsB,EACtB,iBAAiB,EAElB,MAAM,SAAS,CAAC;AAmBjB,KAAK,yBAAyB,GAAG,MAAM,GAAG,8BAA8B,CAAC;AAGzE,qBAAa,gBAAgB;;IAG3B,gBAAgB,CAAC,WAAW,SAAS,8BAA8B,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI;IAKnG,YAAY,CACV,QAAQ,EACJ,yBAAyB,GACzB,QAAQ,CAAC,8BAA8B,CAAC,GAC3C,IAAI;IAKP,gBAAgB,CACd,KAAK,EAAE,iBAAiB,EACxB,GAAG,KAAK,EAAE,yBAAyB,EAAE,GACpC,IAAI;IAKP,gBAAgB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IAKhD,KAAK,IAAI,SAAS;CAGnB;AAED,qBAAa,SAAS;;IACpB,MAAM,CAAC,QAAQ,CAAC,UAAU,gBAAgB;IAU1C,MAAM,CAAC,OAAO,IAAI,gBAAgB;IAIlC,oBAAoB,CAAC,WAAW,SAAS,8BAA8B,EACrE,UAAU,EAAE,WAAW,GACtB,WAAW;IA8Bd,gBAAgB,CACd,QAAQ,EACJ,yBAAyB,GACzB,QAAQ,CAAC,8BAA8B,CAAC,GAC3C,IAAI;IAYP,kBAAkB,CAAC,WAAW,SAAS,8BAA8B,EACnE,UAAU,EAAE,WAAW,EACvB,QAAQ,EAAE,yBAAyB,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC,GACvE,WAAW;IAOd,aAAa,CAAC,WAAW,SAAS,8BAA8B,EAC9D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,yBAAyB,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC,GACvE,WAAW;IAcd,eAAe,CACb,KAAK,EAAE,iBAAiB,EACxB,GAAG,KAAK,EAAE,yBAAyB,EAAE,GACpC,IAAI;IAOP,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IAIhD,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB3C,GAAG,CAAC,WAAW,SAAS,8BAA8B,EACpD,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAMlD,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAC5D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,sBAAsB,CAAC,WAAW,CAAC;IAStC,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAC5D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,iBAAiB,GAAG,SAAS;IAMhC,mBAAmB,CAAC,WAAW,SAAS,8BAA8B,EACpE,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,iBAAiB;IASpB,SAAS,CAAC,WAAW,SAAS,8BAA8B,EAC1D,UAAU,EAAE,WAAW,GACtB,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAUlD,iBAAiB,CAAC,WAAW,SAAS,8BAA8B,EAClE,UAAU,EAAE,WAAW,GACtB,sBAAsB,CAAC,WAAW,CAAC;IAStC,iBAAiB,CAAC,WAAW,SAAS,8BAA8B,EAClE,UAAU,EAAE,WAAW,GACtB,iBAAiB,GAAG,SAAS;IAUhC,MAAM,CAAC,WAAW,SAAS,8BAA8B,EACvD,UAAU,EAAE,WAAW,GACtB,sBAAsB,CAAC,WAAW,CAAC,EAAE;IAmBxC,YAAY,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,SAAS;IASrB,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAC5D,UAAU,EAAE,WAAW,EACvB,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,SAAS;IAIrB,SAAS,CAAC,WAAW,SAAS,8BAA8B,EAC1D,UAAU,EAAE,WAAW,EACvB,IAAI,EAAE,MAAM,GACX,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAKlD,aAAa,CAAC,WAAW,SAAS,8BAA8B,EAC9D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,GAAG,SAAS;IAKrB,YAAY,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,GAAG,SAAS;IAIrB,UAAU,CAAC,WAAW,SAAS,8BAA8B,EAC3D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAK5C,GAAG,CAAC,WAAW,SAAS,8BAA8B,EAC1D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,sBAAsB,CAAC,WAAW,CAAC,GACzC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IA6BzC,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAClE,UAAU,EAAE,WAAW,EACvB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IAyCzC,SAAS,CAAC,WAAW,SAAS,8BAA8B,EAChE,UAAU,EAAE,WAAW,EACvB,KAAK,EAAE,sBAAsB,CAAC,WAAW,CAAC,GACzC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IASzC,MAAM,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,CACP,OAAO,EAAE,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS,KACrD,sBAAsB,CAAC,WAAW,CAAC,GACvC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IAKzC,YAAY,CAAC,WAAW,SAAS,8BAA8B,EACnE,UAAU,EAAE,WAAW,EACvB,OAAO,EAAE,CACP,OAAO,EAAE,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS,KACrD,sBAAsB,CAAC,WAAW,CAAC,GACvC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IAKzC,MAAM,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC;IAmBb,YAAY,CAAC,WAAW,SAAS,8BAA8B,EACnE,UAAU,EAAE,WAAW,GACtB,OAAO,CAAC,OAAO,CAAC;IAUnB,QAAQ,IAAI,iBAAiB,EAAE;CA8ShC"} \ No newline at end of file +{"version":3,"file":"cult-cache.d.ts","sourceRoot":"","sources":["../src/cult-cache.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,8BAA8B,EAC9B,iBAAiB,EACjB,yBAAyB,EAGzB,yBAAyB,EACzB,sBAAsB,EACtB,iBAAiB,EAGlB,MAAM,SAAS,CAAC;AAoBjB,KAAK,yBAAyB,GAAG,MAAM,GAAG,8BAA8B,CAAC;AAGzE,qBAAa,gBAAgB;;IAG3B,gBAAgB,CAAC,WAAW,SAAS,8BAA8B,EAAE,UAAU,EAAE,WAAW,GAAG,IAAI;IAKnG,YAAY,CACV,QAAQ,EACJ,yBAAyB,GACzB,QAAQ,CAAC,8BAA8B,CAAC,GAC3C,IAAI;IAKP,gBAAgB,CACd,KAAK,EAAE,iBAAiB,EACxB,GAAG,KAAK,EAAE,yBAAyB,EAAE,GACpC,IAAI;IAKP,gBAAgB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IAKhD,KAAK,IAAI,SAAS;CAGnB;AAED,qBAAa,SAAS;;IACpB,MAAM,CAAC,QAAQ,CAAC,UAAU,gBAAgB;IAW1C,MAAM,CAAC,OAAO,IAAI,gBAAgB;IAIlC,oBAAoB,CAAC,WAAW,SAAS,8BAA8B,EACrE,UAAU,EAAE,WAAW,GACtB,WAAW;IAsCd,gBAAgB,CACd,QAAQ,EACJ,yBAAyB,GACzB,QAAQ,CAAC,8BAA8B,CAAC,GAC3C,IAAI;IAYP,kBAAkB,CAAC,WAAW,SAAS,8BAA8B,EACnE,UAAU,EAAE,WAAW,EACvB,QAAQ,EAAE,yBAAyB,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC,GACvE,WAAW;IAOd,aAAa,CAAC,WAAW,SAAS,8BAA8B,EAC9D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,yBAAyB,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC,GACvE,WAAW;IAcd,eAAe,CACb,KAAK,EAAE,iBAAiB,EACxB,GAAG,KAAK,EAAE,yBAAyB,EAAE,GACpC,IAAI;IAOP,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI;IAIhD,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IA0B3C,GAAG,CAAC,WAAW,SAAS,8BAA8B,EACpD,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAMlD,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAC5D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,sBAAsB,CAAC,WAAW,CAAC;IAStC,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAC5D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,iBAAiB,GAAG,SAAS;IAMhC,mBAAmB,CAAC,WAAW,SAAS,8BAA8B,EACpE,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,iBAAiB;IASpB,SAAS,CAAC,WAAW,SAAS,8BAA8B,EAC1D,UAAU,EAAE,WAAW,GACtB,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAUlD,iBAAiB,CAAC,WAAW,SAAS,8BAA8B,EAClE,UAAU,EAAE,WAAW,GACtB,sBAAsB,CAAC,WAAW,CAAC;IAStC,iBAAiB,CAAC,WAAW,SAAS,8BAA8B,EAClE,UAAU,EAAE,WAAW,GACtB,iBAAiB,GAAG,SAAS;IAUhC,MAAM,CAAC,WAAW,SAAS,8BAA8B,EACvD,UAAU,EAAE,WAAW,GACtB,sBAAsB,CAAC,WAAW,CAAC,EAAE;IAmBxC,YAAY,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,SAAS;IASrB,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAC5D,UAAU,EAAE,WAAW,EACvB,IAAI,EAAE,MAAM,GACX,MAAM,GAAG,SAAS;IAIrB,SAAS,CAAC,WAAW,SAAS,8BAA8B,EAC1D,UAAU,EAAE,WAAW,EACvB,IAAI,EAAE,MAAM,GACX,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAKlD,aAAa,CAAC,WAAW,SAAS,8BAA8B,EAC9D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,GAAG,SAAS;IAKrB,YAAY,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,GAAG,SAAS;IAIrB,UAAU,CAAC,WAAW,SAAS,8BAA8B,EAC3D,UAAU,EAAE,WAAW,EACvB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS;IAK5C,GAAG,CAAC,WAAW,SAAS,8BAA8B,EAC1D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,sBAAsB,CAAC,WAAW,CAAC,GACzC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IA+BzC,WAAW,CAAC,WAAW,SAAS,8BAA8B,EAClE,UAAU,EAAE,WAAW,EACvB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IA2CzC,SAAS,CAAC,WAAW,SAAS,8BAA8B,EAChE,UAAU,EAAE,WAAW,EACvB,KAAK,EAAE,sBAAsB,CAAC,WAAW,CAAC,GACzC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IASzC,MAAM,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,CACP,OAAO,EAAE,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS,KACrD,sBAAsB,CAAC,WAAW,CAAC,GACvC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IAKzC,YAAY,CAAC,WAAW,SAAS,8BAA8B,EACnE,UAAU,EAAE,WAAW,EACvB,OAAO,EAAE,CACP,OAAO,EAAE,sBAAsB,CAAC,WAAW,CAAC,GAAG,SAAS,KACrD,sBAAsB,CAAC,WAAW,CAAC,GACvC,OAAO,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IAKzC,MAAM,CAAC,WAAW,SAAS,8BAA8B,EAC7D,UAAU,EAAE,WAAW,EACvB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,OAAO,CAAC;IAmBb,YAAY,CAAC,WAAW,SAAS,8BAA8B,EACnE,UAAU,EAAE,WAAW,GACtB,OAAO,CAAC,OAAO,CAAC;IAUnB,QAAQ,IAAI,iBAAiB,EAAE;CAwWhC"} \ No newline at end of file diff --git a/dist/cult-cache.js b/dist/cult-cache.js index 3bedf55..c67480b 100644 --- a/dist/cult-cache.js +++ b/dist/cult-cache.js @@ -28,6 +28,7 @@ exports.CultCacheBuilder = CultCacheBuilder; class CultCache { static GLOBAL_KEY = "__global__"; #definitions = new Map(); + #schemaIdDefinitions = new Map(); #entries = new Map(); #typeEntryIds = new Map(); #nameToKeyMaps = new Map(); @@ -46,6 +47,7 @@ class CultCache { definition, global: definition.global === true, formatter: this.#createFormatter(definition), + catalogEntry: this.#createCatalogEntry(definition), nameAccessor: undefined, indexAccessors: new Map(), }; @@ -57,6 +59,13 @@ class CultCache { registered.indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); } this.#definitions.set(definition.type, registered); + for (const schemaId of registered.catalogEntry.compatibleSchemaIds ?? [registered.catalogEntry.schemaId]) { + const existingBySchema = this.#schemaIdDefinitions.get(schemaId); + if (existingBySchema && existingBySchema !== registered) { + throw new Error(`CultCache schema id "${schemaId}" is already registered for type "${existingBySchema.definition.type}".`); + } + this.#schemaIdDefinitions.set(schemaId, registered); + } this.#rebuildDefinitionLookups(registered); return definition; } @@ -98,9 +107,11 @@ class CultCache { for (const registration of this.#stores) { const entries = await registration.store.pullAll(); for (const entry of entries) { - const registered = this.#definitions.get(entry.type); + const registered = this.#resolveDefinitionForEnvelope(entry); if (!registered) { - throw new Error(`No schema is registered for persisted document type "${entry.type}".`); + throw new Error(entry.schemaId + ? `No schema is registered for persisted schema id "${entry.schemaId}".` + : `No schema is registered for persisted document type "${entry.type}".`); } const payload = this.#cloneBytes(entry.payload); const value = registered.formatter.decode(payload); @@ -207,6 +218,8 @@ class CultCache { type: definition.type, payload: this.#cloneBytes(payload), storedAt: new Date().toISOString(), + schemaId: registered.catalogEntry.schemaId, + catalogEntry: registered.catalogEntry, }; const route = this.#resolveRoute(definition.type); if (!route.primary) { @@ -241,6 +254,8 @@ class CultCache { type: envelope.type, payload, storedAt: envelope.storedAt, + schemaId: envelope.schemaId ?? registered.catalogEntry.schemaId, + catalogEntry: envelope.catalogEntry ?? registered.catalogEntry, }; const route = this.#resolveRoute(definition.type); if (!route.primary) { @@ -315,6 +330,50 @@ class CultCache { }, }; } + #createCatalogEntry(definition) { + const schemaName = definition.schemaName ?? definition.type; + const schemaVersion = definition.schemaVersion ?? `${schemaName}.v1`; + const canonicalSchemaJson = definition.canonicalSchemaJson ?? JSON.stringify({ + schemaName, + schemaVersion, + members: [...(definition.members ?? [])] + .sort((left, right) => left.slot - right.slot) + .map((member) => ({ + slot: member.slot, + name: member.memberName, + type: member.typeName, + isReference: member.isReference === true, + many: member.isMany === true, + targetSchemaName: member.targetSchemaName ?? null, + indexAlias: member.indexAlias ?? null, + isName: member.isName === true, + })), + }); + const schemaId = definition.schemaId ?? schemaName; + const compatibleSchemaIds = [ + ...new Set([schemaId, ...(definition.compatibleSchemaIds ?? [])]), + ]; + return { + schemaId, + schemaName, + schemaVersion, + contentHash: definition.contentHash ?? schemaId, + canonicalSchemaJson, + compatibleSchemaIds, + members: [...(definition.members ?? [])] + .sort((left, right) => left.slot - right.slot) + .map((member) => ({ + slot: member.slot, + memberName: member.memberName, + typeName: member.typeName, + isReference: member.isReference === true, + isMany: member.isMany === true, + targetSchemaName: member.targetSchemaName ?? null, + isName: member.isName === true, + indexAlias: member.indexAlias ?? null, + })), + }; + } #enumerateDefinitionIndexes(definition) { if (!definition.indexes) { return []; @@ -358,6 +417,12 @@ class CultCache { } return registered; } + #resolveDefinitionForEnvelope(entry) { + if (entry.schemaId) { + return this.#schemaIdDefinitions.get(entry.schemaId); + } + return this.#definitions.get(entry.type); + } #resolveRoute(type) { const typeSpecific = this.#stores.filter((registration) => registration.types.includes(type)); if (typeSpecific.length > 0) { @@ -389,6 +454,8 @@ class CultCache { const hydrated = { ...entry, payload: this.#cloneBytes(entry.payload), + schemaId: entry.schemaId ?? registered.catalogEntry.schemaId, + catalogEntry: entry.catalogEntry ?? registered.catalogEntry, value, }; this.#entries.set(entryId, hydrated); @@ -514,6 +581,8 @@ class CultCache { type: entry.type, payload: this.#cloneBytes(entry.payload), storedAt: entry.storedAt, + schemaId: entry.schemaId, + catalogEntry: entry.catalogEntry, }; } #resetHydratedState() { diff --git a/dist/document.d.ts.map b/dist/document.d.ts.map index 0d5cff2..c2abff8 100644 --- a/dist/document.d.ts.map +++ b/dist/document.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../src/document.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,8BAA8B,EAC9B,2BAA2B,EAC3B,yBAAyB,EACzB,eAAe,EAChB,MAAM,SAAS,CAAC;AAEjB,wBAAgB,kBAAkB,CAAC,OAAO,SAAS,eAAe,EAChE,UAAU,EAAE,2BAA2B,CAAC,OAAO,CAAC,GAC/C,2BAA2B,CAAC,OAAO,CAAC,CA6BtC;AAED,wBAAgB,sBAAsB,CACpC,YAAY,SAAS,SAAS,8BAA8B,EAAE,EAC9D,GAAG,WAAW,EAAE,YAAY,GAAG,yBAAyB,CAAC,YAAY,CAAC,CAIvE"} \ No newline at end of file +{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../src/document.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,8BAA8B,EAC9B,2BAA2B,EAC3B,yBAAyB,EACzB,eAAe,EAChB,MAAM,SAAS,CAAC;AAEjB,wBAAgB,kBAAkB,CAAC,OAAO,SAAS,eAAe,EAChE,UAAU,EAAE,2BAA2B,CAAC,OAAO,CAAC,GAC/C,2BAA2B,CAAC,OAAO,CAAC,CA4DtC;AAED,wBAAgB,sBAAsB,CACpC,YAAY,SAAS,SAAS,8BAA8B,EAAE,EAC9D,GAAG,WAAW,EAAE,YAAY,GAAG,yBAAyB,CAAC,YAAY,CAAC,CAIvE"} \ No newline at end of file diff --git a/dist/document.js b/dist/document.js index b9fe238..5b56ceb 100644 --- a/dist/document.js +++ b/dist/document.js @@ -6,6 +6,17 @@ function defineDocumentType(definition) { if (!definition.type || definition.type.trim().length === 0) { throw new Error("CultCache document types must declare a non-empty type."); } + for (const [fieldName, fieldValue] of [ + ["schemaId", definition.schemaId], + ["schemaName", definition.schemaName], + ["schemaVersion", definition.schemaVersion], + ["contentHash", definition.contentHash], + ["canonicalSchemaJson", definition.canonicalSchemaJson], + ]) { + if (fieldValue !== undefined && fieldValue.trim().length === 0) { + throw new Error(`CultCache document type "${definition.type}" declares an empty ${fieldName}.`); + } + } if (definition.name !== undefined && typeof definition.name !== "string" && typeof definition.name !== "function") { throw new Error(`CultCache document type "${definition.type}" declares an invalid name accessor.`); } @@ -28,6 +39,24 @@ function defineDocumentType(definition) { } } } + if (definition.members) { + const seenSlots = new Set(); + for (const member of definition.members) { + if (!Number.isInteger(member.slot) || member.slot < 0) { + throw new Error(`CultCache document type "${definition.type}" declares an invalid schema member slot.`); + } + if (seenSlots.has(member.slot)) { + throw new Error(`CultCache document type "${definition.type}" declares duplicate schema member slot ${member.slot}.`); + } + seenSlots.add(member.slot); + if (!member.memberName || member.memberName.trim().length === 0) { + throw new Error(`CultCache document type "${definition.type}" declares a schema member with an empty name.`); + } + if (!member.typeName || member.typeName.trim().length === 0) { + throw new Error(`CultCache document type "${definition.type}" declares schema member "${member.memberName}" with an empty type.`); + } + } + } return Object.freeze({ ...definition }); } function defineDocumentRegistry(...definitions) { diff --git a/dist/index.d.ts b/dist/index.d.ts index 0f10e92..4f74714 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,5 +1,5 @@ export { CultCache, CultCacheBuilder } from "./cult-cache"; export { defineDocumentRegistry, defineDocumentType } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; -export type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheDocumentAccessor, CultCacheDocumentDefinition, CultCacheDocumentFieldName, CultCacheDocumentFormatter, CultCacheDocumentIndexDefinition, CultCacheDocumentRegistry, CultCacheSchema, CultCacheDocumentValue, CultCacheEnvelope, CultCacheIndexScalar, InferCultCacheSchemaValue, CultCacheStoreRegistration, PushAllOptions, } from "./types"; +export type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheDocumentAccessor, CultCacheDocumentDefinition, CultCacheDocumentFieldName, CultCacheDocumentFormatter, CultCacheDocumentIndexDefinition, CultCacheDocumentRegistry, CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember, CultCacheSchema, CultCacheDocumentValue, CultCacheEnvelope, CultCacheIndexScalar, InferCultCacheSchemaValue, CultCacheStoreRegistration, PushAllOptions, } from "./types"; //# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/dist/index.d.ts.map b/dist/index.d.ts.map index 5015a57..e3ea665 100644 --- a/dist/index.d.ts.map +++ b/dist/index.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,iCAAiC,EAAE,MAAM,yCAAyC,CAAC;AAC5F,YAAY,EACV,8BAA8B,EAC9B,iBAAiB,EACjB,yBAAyB,EACzB,2BAA2B,EAC3B,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,yBAAyB,EACzB,eAAe,EACf,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,yBAAyB,EACzB,0BAA0B,EAC1B,cAAc,GACf,MAAM,SAAS,CAAC"} \ No newline at end of file +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,iCAAiC,EAAE,MAAM,yCAAyC,CAAC;AAC5F,YAAY,EACV,8BAA8B,EAC9B,iBAAiB,EACjB,yBAAyB,EACzB,2BAA2B,EAC3B,0BAA0B,EAC1B,0BAA0B,EAC1B,gCAAgC,EAChC,yBAAyB,EACzB,2BAA2B,EAC3B,4BAA4B,EAC5B,eAAe,EACf,sBAAsB,EACtB,iBAAiB,EACjB,oBAAoB,EACpB,yBAAyB,EACzB,0BAA0B,EAC1B,cAAc,GACf,MAAM,SAAS,CAAC"} \ No newline at end of file diff --git a/dist/single-file-messagepack-backing-store.d.ts.map b/dist/single-file-messagepack-backing-store.d.ts.map index 172b69a..55dce32 100644 --- a/dist/single-file-messagepack-backing-store.d.ts.map +++ b/dist/single-file-messagepack-backing-store.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"single-file-messagepack-backing-store.d.ts","sourceRoot":"","sources":["../src/single-file-messagepack-backing-store.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAmBpF,qBAAa,iCAAkC,YAAW,iBAAiB;;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAId,QAAQ,EAAE,MAAM;IAItB,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAiCvC,IAAI,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAW7C,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/C,OAAO,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;CAiDzF"} \ No newline at end of file +{"version":3,"file":"single-file-messagepack-backing-store.d.ts","sourceRoot":"","sources":["../src/single-file-messagepack-backing-store.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,iBAAiB,EACjB,iBAAiB,EAGjB,cAAc,EACf,MAAM,SAAS,CAAC;AAsBjB,qBAAa,iCAAkC,YAAW,iBAAiB;;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAId,QAAQ,EAAE,MAAM;IAItB,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;IA6DvC,IAAI,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAW7C,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/C,OAAO,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;CAiDzF"} \ No newline at end of file diff --git a/dist/single-file-messagepack-backing-store.js b/dist/single-file-messagepack-backing-store.js index 38023b4..e95d270 100644 --- a/dist/single-file-messagepack-backing-store.js +++ b/dist/single-file-messagepack-backing-store.js @@ -5,11 +5,13 @@ const promises_1 = require("node:fs/promises"); const node_path_1 = require("node:path"); const msgpack_1 = require("@msgpack/msgpack"); const zod_1 = require("zod"); +const STORE_FORMAT_VERSION = "cultcache.store.v1"; const envelopeSchema = zod_1.z.object({ key: zod_1.z.string().min(1), type: zod_1.z.string().min(1), payload: zod_1.z.instanceof(Uint8Array), storedAt: zod_1.z.string().min(1), + schemaId: zod_1.z.string().min(1).optional(), }); const envelopeArraySchema = zod_1.z.array(envelopeSchema); const legacyEnvelopeArraySchema = zod_1.z.array(zod_1.z.object({ @@ -27,23 +29,47 @@ class SingleFileMessagePackBackingStore { async pullAll() { try { const data = await (0, promises_1.readFile)(this.filePath); - const decoded = legacyEnvelopeArraySchema.parse((0, msgpack_1.decode)(data)); - let repairedLegacyPayload = false; - const normalized = decoded.map((entry) => { - const payload = normalizePayload(entry.payload); - if (payload !== entry.payload) { - repairedLegacyPayload = true; - } - return { - ...entry, - payload, - }; - }); - const parsed = envelopeArraySchema.parse(normalized); - if (repairedLegacyPayload) { - await this.#writeAll(parsed); + if (data.length === 0) { + return []; } - return parsed; + const decoded = (0, msgpack_1.decode)(data); + const snapshot = decodeSnapshot(decoded); + if (snapshot) { + return snapshot.records.map((record) => { + const catalogEntry = snapshot.catalogBySchemaId.get(record.schemaId); + if (!catalogEntry) { + throw new Error(`CultCache persisted record "${record.key}" references missing schema id "${record.schemaId}".`); + } + return { + key: record.key, + type: catalogEntry.schemaName, + schemaId: record.schemaId, + catalogEntry, + payload: record.payload, + storedAt: record.storedAt, + }; + }); + } + const legacy = decodeLegacyEnvelopeArray(decoded); + if (legacy) { + let repairedLegacyPayload = false; + const normalized = legacy.map((entry) => { + const payload = normalizePayload(entry.payload); + if (payload !== entry.payload) { + repairedLegacyPayload = true; + } + return { + ...entry, + payload, + }; + }); + const parsed = envelopeArraySchema.parse(normalized); + if (repairedLegacyPayload) { + await this.#writeAll(parsed); + } + return parsed; + } + throw new Error(`CultCache file ${this.filePath} is not a recognized CultCache MessagePack store.`); } catch (error) { const code = error.code; @@ -100,7 +126,7 @@ class SingleFileMessagePackBackingStore { .toString(36) .slice(2)}`; try { - await (0, promises_1.writeFile)(tempPath, (0, msgpack_1.encode)(entries)); + await (0, promises_1.writeFile)(tempPath, (0, msgpack_1.encode)(encodeSnapshot(entries))); await (0, promises_1.rename)(tempPath, this.filePath); } catch (error) { @@ -110,6 +136,155 @@ class SingleFileMessagePackBackingStore { } } exports.SingleFileMessagePackBackingStore = SingleFileMessagePackBackingStore; +function encodeSnapshot(entries) { + const catalog = [...catalogEntriesFor(entries).values()] + .sort((left, right) => compareOrdinal(left.schemaName, right.schemaName)); + const records = [...entries] + .sort((left, right) => compareOrdinal(left.key, right.key)) + .map((entry) => [ + entry.key, + schemaIdForEnvelope(entry), + entry.storedAt, + entry.payload, + ]); + return [ + STORE_FORMAT_VERSION, + catalog.map(encodeCatalogEntry), + records, + ]; +} +function catalogEntriesFor(entries) { + const catalog = new Map(); + for (const entry of entries) { + const schemaId = schemaIdForEnvelope(entry); + if (!catalog.has(schemaId)) { + catalog.set(schemaId, entry.catalogEntry ?? { + schemaId, + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + contentHash: schemaId, + canonicalSchemaJson: JSON.stringify({ + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + members: [], + }), + compatibleSchemaIds: [schemaId], + members: [], + }); + } + } + return catalog; +} +function schemaIdForEnvelope(entry) { + return entry.schemaId ?? entry.type; +} +function encodeCatalogEntry(entry) { + return [ + entry.schemaId, + entry.schemaName, + entry.schemaVersion, + entry.contentHash, + entry.canonicalSchemaJson, + [...(entry.compatibleSchemaIds ?? [entry.schemaId])], + [...(entry.members ?? [])].map(encodeCatalogMember), + ]; +} +function encodeCatalogMember(member) { + return [ + member.slot, + member.memberName, + member.typeName, + member.isReference === true, + member.isMany === true, + member.targetSchemaName ?? null, + member.isName === true, + member.indexAlias ?? null, + ]; +} +function decodeSnapshot(decoded) { + if (!Array.isArray(decoded) || decoded.length === 0 || decoded[0] !== STORE_FORMAT_VERSION) { + return undefined; + } + const catalogRaw = decoded[1]; + const recordsRaw = decoded[2]; + if (!Array.isArray(catalogRaw) || !Array.isArray(recordsRaw)) { + throw new Error("CultCache v1 snapshot must contain a schema catalog and record array."); + } + const catalogBySchemaId = new Map(); + for (const entry of catalogRaw) { + const catalogEntry = decodeCatalogEntry(entry); + catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry); + for (const compatibleSchemaId of catalogEntry.compatibleSchemaIds ?? []) { + catalogBySchemaId.set(compatibleSchemaId, catalogEntry); + } + } + const records = recordsRaw.map(decodeRecord); + return { catalogBySchemaId, records }; +} +function decodeCatalogEntry(value) { + if (!Array.isArray(value)) { + throw new Error("CultCache schema catalog entries must be MessagePack arrays."); + } + const [schemaId = "", schemaName = "", schemaVersion = "", contentHash = "", canonicalSchemaJson = "", compatibleSchemaIds = [], members = [],] = value; + if (!isNonEmptyString(schemaId) || !isNonEmptyString(schemaName)) { + throw new Error("CultCache schema catalog entries must declare schemaId and schemaName."); + } + if (!Array.isArray(compatibleSchemaIds) || !compatibleSchemaIds.every(isNonEmptyString)) { + throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid compatible schema ids.`); + } + if (!Array.isArray(members)) { + throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid members.`); + } + return { + schemaId, + schemaName, + schemaVersion: isNonEmptyString(schemaVersion) ? schemaVersion : `${schemaName}.v1`, + contentHash: isNonEmptyString(contentHash) ? contentHash : schemaId, + canonicalSchemaJson: typeof canonicalSchemaJson === "string" ? canonicalSchemaJson : "", + compatibleSchemaIds, + members: members.map(decodeCatalogMember), + }; +} +function decodeCatalogMember(value) { + if (!Array.isArray(value)) { + throw new Error("CultCache schema catalog members must be MessagePack arrays."); + } + const [slot = -1, memberName = "", typeName = "", isReference = false, isMany = false, targetSchemaName = null, isName = false, indexAlias = null,] = value; + if (!Number.isInteger(slot) || slot < 0 || !isNonEmptyString(memberName) || !isNonEmptyString(typeName)) { + throw new Error("CultCache schema catalog member has invalid slot, name, or type."); + } + return { + slot, + memberName, + typeName, + isReference: isReference === true, + isMany: isMany === true, + targetSchemaName: typeof targetSchemaName === "string" ? targetSchemaName : null, + isName: isName === true, + indexAlias: typeof indexAlias === "string" ? indexAlias : null, + }; +} +function decodeRecord(value) { + if (!Array.isArray(value)) { + throw new Error("CultCache persisted records must be MessagePack arrays."); + } + const [key = "", schemaId = "", storedAt = "", payload = new Uint8Array()] = value; + if (!isNonEmptyString(key) || !isNonEmptyString(schemaId) || !isNonEmptyString(storedAt)) { + throw new Error("CultCache persisted records must declare key, schemaId, and storedAt."); + } + return { + key, + schemaId, + storedAt, + payload: normalizePayload(payload), + }; +} +function decodeLegacyEnvelopeArray(decoded) { + if (!Array.isArray(decoded)) { + return undefined; + } + return legacyEnvelopeArraySchema.parse(decoded); +} function normalizePayload(payload) { if (payload instanceof Uint8Array) { return payload; @@ -128,3 +303,9 @@ function normalizePayload(payload) { function isObject(value) { return typeof value === "object" && value !== null; } +function isNonEmptyString(value) { + return typeof value === "string" && value.length > 0; +} +function compareOrdinal(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} diff --git a/dist/types.d.ts b/dist/types.d.ts index 8f485cf..c111ca5 100644 --- a/dist/types.d.ts +++ b/dist/types.d.ts @@ -9,6 +9,25 @@ export interface CultCacheDocumentFormatter { encode(value: TValue): Uint8Array; decode(payload: Uint8Array): TValue; } +export interface CultCacheSchemaCatalogMember { + slot: number; + memberName: string; + typeName: string; + isReference?: boolean; + isMany?: boolean; + targetSchemaName?: string | null; + isName?: boolean; + indexAlias?: string | null; +} +export interface CultCacheSchemaCatalogEntry { + schemaId: string; + schemaName: string; + schemaVersion: string; + contentHash: string; + canonicalSchemaJson: string; + compatibleSchemaIds?: readonly string[]; + members?: readonly CultCacheSchemaCatalogMember[]; +} export interface CultCacheDocumentIndexDefinition { name: string; accessor: CultCacheDocumentAccessor; @@ -16,7 +35,13 @@ export interface CultCacheDocumentIndexDefinition { export interface CultCacheDocumentDefinition { type: string; schema: TSchema; + schemaId?: string; schemaName?: string; + schemaVersion?: string; + contentHash?: string; + canonicalSchemaJson?: string; + compatibleSchemaIds?: readonly string[]; + members?: readonly CultCacheSchemaCatalogMember[]; global?: boolean; name?: CultCacheDocumentAccessor>; indexes?: Readonly>>> | readonly CultCacheDocumentIndexDefinition>[]; @@ -32,6 +57,8 @@ export interface CultCacheEnvelope { type: string; payload: Uint8Array; storedAt: string; + schemaId?: string; + catalogEntry?: CultCacheSchemaCatalogEntry; } export interface PushAllOptions { soft?: boolean; diff --git a/dist/types.d.ts.map b/dist/types.d.ts.map index 2d07804..abafae1 100644 --- a/dist/types.d.ts.map +++ b/dist/types.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAEtE,MAAM,WAAW,eAAe,CAAC,MAAM,GAAG,OAAO;IAC/C,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,yBAAyB,CAAC,OAAO,SAAS,eAAe,IACnE,OAAO,SAAS,eAAe,CAAC,MAAM,MAAM,CAAC,GAAG,MAAM,GAAG,KAAK,CAAC;AAEjE,MAAM,MAAM,0BAA0B,CAAC,MAAM,IAAI,MAAM,SAAS,MAAM,GAAG,OAAO,CAAC,MAAM,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;AAE/G,MAAM,MAAM,yBAAyB,CAAC,MAAM,IACxC,0BAA0B,CAAC,MAAM,CAAC,GAClC,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,oBAAoB,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;AAEjE,MAAM,WAAW,0BAA0B,CAAC,MAAM;IAChD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAClC,MAAM,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,gCAAgC,CAAC,MAAM;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,yBAAyB,CAAC,MAAM,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,2BAA2B,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe;IAC5F,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,yBAAyB,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,OAAO,CAAC,EACJ,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,yBAAyB,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GACvF,SAAS,gCAAgC,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;IACpF,SAAS,CAAC,EAAE,0BAA0B,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC;CAC5E;AAED,MAAM,MAAM,sBAAsB,CAAC,WAAW,SAAS,2BAA2B,IAAI,yBAAyB,CAC7G,WAAW,CAAC,QAAQ,CAAC,CACtB,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG,2BAA2B,CAAC,GAAG,CAAC,CAAC;AAE9E,MAAM,WAAW,yBAAyB,CACxC,YAAY,SAAS,SAAS,8BAA8B,EAAE,GAAG,SAAS,8BAA8B,EAAE;IAE1G,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,UAAU,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,iBAAiB,CAAC;IACzB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB"} \ No newline at end of file +{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAEtE,MAAM,WAAW,eAAe,CAAC,MAAM,GAAG,OAAO;IAC/C,KAAK,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,yBAAyB,CAAC,OAAO,SAAS,eAAe,IACnE,OAAO,SAAS,eAAe,CAAC,MAAM,MAAM,CAAC,GAAG,MAAM,GAAG,KAAK,CAAC;AAEjE,MAAM,MAAM,0BAA0B,CAAC,MAAM,IAAI,MAAM,SAAS,MAAM,GAAG,OAAO,CAAC,MAAM,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;AAE/G,MAAM,MAAM,yBAAyB,CAAC,MAAM,IACxC,0BAA0B,CAAC,MAAM,CAAC,GAClC,CAAC,CAAC,KAAK,EAAE,MAAM,KAAK,oBAAoB,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;AAEjE,MAAM,WAAW,0BAA0B,CAAC,MAAM;IAChD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC;IAClC,MAAM,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,OAAO,CAAC,EAAE,SAAS,4BAA4B,EAAE,CAAC;CACnD;AAED,MAAM,WAAW,gCAAgC,CAAC,MAAM;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,yBAAyB,CAAC,MAAM,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,2BAA2B,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe;IAC5F,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,OAAO,CAAC,EAAE,SAAS,4BAA4B,EAAE,CAAC;IAClD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,yBAAyB,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,OAAO,CAAC,EACJ,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,yBAAyB,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GACvF,SAAS,gCAAgC,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;IACpF,SAAS,CAAC,EAAE,0BAA0B,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC;CAC5E;AAED,MAAM,MAAM,sBAAsB,CAAC,WAAW,SAAS,2BAA2B,IAAI,yBAAyB,CAC7G,WAAW,CAAC,QAAQ,CAAC,CACtB,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG,2BAA2B,CAAC,GAAG,CAAC,CAAC;AAE9E,MAAM,WAAW,yBAAyB,CACxC,YAAY,SAAS,SAAS,8BAA8B,EAAE,GAAG,SAAS,8BAA8B,EAAE;IAE1G,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,UAAU,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,2BAA2B,CAAC;CAC5C;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,iBAAiB,CAAC;IACzB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB"} \ No newline at end of file diff --git a/src/cult-cache.ts b/src/cult-cache.ts index da7294f..5f3f691 100644 --- a/src/cult-cache.ts +++ b/src/cult-cache.ts @@ -9,6 +9,7 @@ import type { CultCacheDocumentRegistry, CultCacheDocumentValue, CultCacheEnvelope, + CultCacheSchemaCatalogEntry, CultCacheStoreRegistration, } from "./types"; @@ -21,6 +22,7 @@ type RegisteredDefinition = { readonly definition: AnyCultCacheDocumentDefinition; readonly global: boolean; readonly formatter: CultCacheDocumentFormatter; + readonly catalogEntry: CultCacheSchemaCatalogEntry; nameAccessor?: (value: unknown) => string | undefined; readonly indexAccessors: Map string | undefined>; }; @@ -71,6 +73,7 @@ export class CultCache { static readonly GLOBAL_KEY = "__global__"; readonly #definitions = new Map(); + readonly #schemaIdDefinitions = new Map(); readonly #entries = new Map(); readonly #typeEntryIds = new Map>(); readonly #nameToKeyMaps = new Map>(); @@ -94,6 +97,7 @@ export class CultCache { definition, global: definition.global === true, formatter: this.#createFormatter(definition), + catalogEntry: this.#createCatalogEntry(definition), nameAccessor: undefined, indexAccessors: new Map string | undefined>(), }; @@ -110,6 +114,13 @@ export class CultCache { } this.#definitions.set(definition.type, registered); + for (const schemaId of registered.catalogEntry.compatibleSchemaIds ?? [registered.catalogEntry.schemaId]) { + const existingBySchema = this.#schemaIdDefinitions.get(schemaId); + if (existingBySchema && existingBySchema !== registered) { + throw new Error(`CultCache schema id "${schemaId}" is already registered for type "${existingBySchema.definition.type}".`); + } + this.#schemaIdDefinitions.set(schemaId, registered); + } this.#rebuildDefinitionLookups(registered); return definition; } @@ -179,9 +190,13 @@ export class CultCache { const entries = await registration.store.pullAll(); for (const entry of entries) { - const registered = this.#definitions.get(entry.type); + const registered = this.#resolveDefinitionForEnvelope(entry); if (!registered) { - throw new Error(`No schema is registered for persisted document type "${entry.type}".`); + throw new Error( + entry.schemaId + ? `No schema is registered for persisted schema id "${entry.schemaId}".` + : `No schema is registered for persisted document type "${entry.type}".`, + ); } const payload = this.#cloneBytes(entry.payload); @@ -358,6 +373,8 @@ export class CultCache { type: definition.type, payload: this.#cloneBytes(payload), storedAt: new Date().toISOString(), + schemaId: registered.catalogEntry.schemaId, + catalogEntry: registered.catalogEntry, }; const route = this.#resolveRoute(definition.type); @@ -402,6 +419,8 @@ export class CultCache { type: envelope.type, payload, storedAt: envelope.storedAt, + schemaId: envelope.schemaId ?? registered.catalogEntry.schemaId, + catalogEntry: envelope.catalogEntry ?? registered.catalogEntry, }; const route = this.#resolveRoute(definition.type); @@ -511,6 +530,52 @@ export class CultCache { }; } + #createCatalogEntry(definition: AnyCultCacheDocumentDefinition): CultCacheSchemaCatalogEntry { + const schemaName = definition.schemaName ?? definition.type; + const schemaVersion = definition.schemaVersion ?? `${schemaName}.v1`; + const canonicalSchemaJson = definition.canonicalSchemaJson ?? JSON.stringify({ + schemaName, + schemaVersion, + members: [...(definition.members ?? [])] + .sort((left, right) => left.slot - right.slot) + .map((member) => ({ + slot: member.slot, + name: member.memberName, + type: member.typeName, + isReference: member.isReference === true, + many: member.isMany === true, + targetSchemaName: member.targetSchemaName ?? null, + indexAlias: member.indexAlias ?? null, + isName: member.isName === true, + })), + }); + const schemaId = definition.schemaId ?? schemaName; + const compatibleSchemaIds = [ + ...new Set([schemaId, ...(definition.compatibleSchemaIds ?? [])]), + ]; + + return { + schemaId, + schemaName, + schemaVersion, + contentHash: definition.contentHash ?? schemaId, + canonicalSchemaJson, + compatibleSchemaIds, + members: [...(definition.members ?? [])] + .sort((left, right) => left.slot - right.slot) + .map((member) => ({ + slot: member.slot, + memberName: member.memberName, + typeName: member.typeName, + isReference: member.isReference === true, + isMany: member.isMany === true, + targetSchemaName: member.targetSchemaName ?? null, + isName: member.isName === true, + indexAlias: member.indexAlias ?? null, + })), + }; + } + #enumerateDefinitionIndexes( definition: AnyCultCacheDocumentDefinition, ): Array<[string, AccessorInput]> { @@ -576,6 +641,14 @@ export class CultCache { return registered; } + #resolveDefinitionForEnvelope(entry: CultCacheEnvelope): RegisteredDefinition | undefined { + if (entry.schemaId) { + return this.#schemaIdDefinitions.get(entry.schemaId); + } + + return this.#definitions.get(entry.type); + } + #resolveRoute(type: string): StoreRoute { const typeSpecific = this.#stores.filter((registration) => registration.types.includes(type)); if (typeSpecific.length > 0) { @@ -618,6 +691,8 @@ export class CultCache { const hydrated: HydratedEntry = { ...entry, payload: this.#cloneBytes(entry.payload), + schemaId: entry.schemaId ?? registered.catalogEntry.schemaId, + catalogEntry: entry.catalogEntry ?? registered.catalogEntry, value, }; this.#entries.set(entryId, hydrated); @@ -769,6 +844,8 @@ export class CultCache { type: entry.type, payload: this.#cloneBytes(entry.payload), storedAt: entry.storedAt, + schemaId: entry.schemaId, + catalogEntry: entry.catalogEntry, }; } diff --git a/src/document.ts b/src/document.ts index ce4d103..ff85d5a 100644 --- a/src/document.ts +++ b/src/document.ts @@ -12,6 +12,18 @@ export function defineDocumentType( throw new Error("CultCache document types must declare a non-empty type."); } + for (const [fieldName, fieldValue] of [ + ["schemaId", definition.schemaId], + ["schemaName", definition.schemaName], + ["schemaVersion", definition.schemaVersion], + ["contentHash", definition.contentHash], + ["canonicalSchemaJson", definition.canonicalSchemaJson], + ] as const) { + if (fieldValue !== undefined && fieldValue.trim().length === 0) { + throw new Error(`CultCache document type "${definition.type}" declares an empty ${fieldName}.`); + } + } + if (definition.name !== undefined && typeof definition.name !== "string" && typeof definition.name !== "function") { throw new Error(`CultCache document type "${definition.type}" declares an invalid name accessor.`); } @@ -35,6 +47,25 @@ export function defineDocumentType( } } + if (definition.members) { + const seenSlots = new Set(); + for (const member of definition.members) { + if (!Number.isInteger(member.slot) || member.slot < 0) { + throw new Error(`CultCache document type "${definition.type}" declares an invalid schema member slot.`); + } + if (seenSlots.has(member.slot)) { + throw new Error(`CultCache document type "${definition.type}" declares duplicate schema member slot ${member.slot}.`); + } + seenSlots.add(member.slot); + if (!member.memberName || member.memberName.trim().length === 0) { + throw new Error(`CultCache document type "${definition.type}" declares a schema member with an empty name.`); + } + if (!member.typeName || member.typeName.trim().length === 0) { + throw new Error(`CultCache document type "${definition.type}" declares schema member "${member.memberName}" with an empty type.`); + } + } + } + return Object.freeze({ ...definition }); } diff --git a/src/index.ts b/src/index.ts index fac6574..07d1955 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,8 @@ export type { CultCacheDocumentFormatter, CultCacheDocumentIndexDefinition, CultCacheDocumentRegistry, + CultCacheSchemaCatalogEntry, + CultCacheSchemaCatalogMember, CultCacheSchema, CultCacheDocumentValue, CultCacheEnvelope, diff --git a/src/single-file-messagepack-backing-store.ts b/src/single-file-messagepack-backing-store.ts index eed4229..3ed8921 100644 --- a/src/single-file-messagepack-backing-store.ts +++ b/src/single-file-messagepack-backing-store.ts @@ -4,13 +4,22 @@ import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; -import type { CacheBackingStore, CultCacheEnvelope, PushAllOptions } from "./types"; +import type { + CacheBackingStore, + CultCacheEnvelope, + CultCacheSchemaCatalogEntry, + CultCacheSchemaCatalogMember, + PushAllOptions, +} from "./types"; + +const STORE_FORMAT_VERSION = "cultcache.store.v1"; const envelopeSchema = z.object({ key: z.string().min(1), type: z.string().min(1), payload: z.instanceof(Uint8Array), storedAt: z.string().min(1), + schemaId: z.string().min(1).optional(), }); const envelopeArraySchema = z.array(envelopeSchema); @@ -35,26 +44,54 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { async pullAll(): Promise { try { const data = await readFile(this.filePath); - const decoded = legacyEnvelopeArraySchema.parse(decode(data)); - let repairedLegacyPayload = false; - const normalized = decoded.map((entry) => { - const payload = normalizePayload(entry.payload); - if (payload !== entry.payload) { - repairedLegacyPayload = true; - } - - return { - ...entry, - payload, - }; - }); - const parsed = envelopeArraySchema.parse(normalized) as CultCacheEnvelope[]; - - if (repairedLegacyPayload) { - await this.#writeAll(parsed); + if (data.length === 0) { + return []; } - return parsed; + const decoded = decode(data); + const snapshot = decodeSnapshot(decoded); + if (snapshot) { + return snapshot.records.map((record) => { + const catalogEntry = snapshot.catalogBySchemaId.get(record.schemaId); + if (!catalogEntry) { + throw new Error(`CultCache persisted record "${record.key}" references missing schema id "${record.schemaId}".`); + } + + return { + key: record.key, + type: catalogEntry.schemaName, + schemaId: record.schemaId, + catalogEntry, + payload: record.payload, + storedAt: record.storedAt, + }; + }); + } + + const legacy = decodeLegacyEnvelopeArray(decoded); + if (legacy) { + let repairedLegacyPayload = false; + const normalized = legacy.map((entry) => { + const payload = normalizePayload(entry.payload); + if (payload !== entry.payload) { + repairedLegacyPayload = true; + } + + return { + ...entry, + payload, + }; + }); + const parsed = envelopeArraySchema.parse(normalized) as CultCacheEnvelope[]; + + if (repairedLegacyPayload) { + await this.#writeAll(parsed); + } + + return parsed; + } + + throw new Error(`CultCache file ${this.filePath} is not a recognized CultCache MessagePack store.`); } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "ENOENT") { @@ -128,7 +165,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { .slice(2)}`; try { - await writeFile(tempPath, encode(entries)); + await writeFile(tempPath, encode(encodeSnapshot(entries))); await rename(tempPath, this.filePath); } catch (error) { await rm(tempPath, { force: true }).catch(() => undefined); @@ -137,6 +174,222 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } } +type PersistedRecord = { + key: string; + schemaId: string; + storedAt: string; + payload: Uint8Array; +}; + +type DecodedSnapshot = { + catalogBySchemaId: Map; + records: PersistedRecord[]; +}; + +function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] { + const catalog = [...catalogEntriesFor(entries).values()] + .sort((left, right) => compareOrdinal(left.schemaName, right.schemaName)); + const records = [...entries] + .sort((left, right) => compareOrdinal(left.key, right.key)) + .map((entry) => [ + entry.key, + schemaIdForEnvelope(entry), + entry.storedAt, + entry.payload, + ]); + + return [ + STORE_FORMAT_VERSION, + catalog.map(encodeCatalogEntry), + records, + ]; +} + +function catalogEntriesFor(entries: CultCacheEnvelope[]): Map { + const catalog = new Map(); + for (const entry of entries) { + const schemaId = schemaIdForEnvelope(entry); + if (!catalog.has(schemaId)) { + catalog.set(schemaId, entry.catalogEntry ?? { + schemaId, + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + contentHash: schemaId, + canonicalSchemaJson: JSON.stringify({ + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + members: [], + }), + compatibleSchemaIds: [schemaId], + members: [], + }); + } + } + + return catalog; +} + +function schemaIdForEnvelope(entry: CultCacheEnvelope): string { + return entry.schemaId ?? entry.type; +} + +function encodeCatalogEntry(entry: CultCacheSchemaCatalogEntry): unknown[] { + return [ + entry.schemaId, + entry.schemaName, + entry.schemaVersion, + entry.contentHash, + entry.canonicalSchemaJson, + [...(entry.compatibleSchemaIds ?? [entry.schemaId])], + [...(entry.members ?? [])].map(encodeCatalogMember), + ]; +} + +function encodeCatalogMember(member: CultCacheSchemaCatalogMember): unknown[] { + return [ + member.slot, + member.memberName, + member.typeName, + member.isReference === true, + member.isMany === true, + member.targetSchemaName ?? null, + member.isName === true, + member.indexAlias ?? null, + ]; +} + +function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined { + if (!Array.isArray(decoded) || decoded.length === 0 || decoded[0] !== STORE_FORMAT_VERSION) { + return undefined; + } + + const catalogRaw = decoded[1]; + const recordsRaw = decoded[2]; + if (!Array.isArray(catalogRaw) || !Array.isArray(recordsRaw)) { + throw new Error("CultCache v1 snapshot must contain a schema catalog and record array."); + } + + const catalogBySchemaId = new Map(); + for (const entry of catalogRaw) { + const catalogEntry = decodeCatalogEntry(entry); + catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry); + for (const compatibleSchemaId of catalogEntry.compatibleSchemaIds ?? []) { + catalogBySchemaId.set(compatibleSchemaId, catalogEntry); + } + } + + const records = recordsRaw.map(decodeRecord); + return { catalogBySchemaId, records }; +} + +function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry { + if (!Array.isArray(value)) { + throw new Error("CultCache schema catalog entries must be MessagePack arrays."); + } + + const [ + schemaId = "", + schemaName = "", + schemaVersion = "", + contentHash = "", + canonicalSchemaJson = "", + compatibleSchemaIds = [], + members = [], + ] = value; + + if (!isNonEmptyString(schemaId) || !isNonEmptyString(schemaName)) { + throw new Error("CultCache schema catalog entries must declare schemaId and schemaName."); + } + + if (!Array.isArray(compatibleSchemaIds) || !compatibleSchemaIds.every(isNonEmptyString)) { + throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid compatible schema ids.`); + } + + if (!Array.isArray(members)) { + throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid members.`); + } + + return { + schemaId, + schemaName, + schemaVersion: isNonEmptyString(schemaVersion) ? schemaVersion : `${schemaName}.v1`, + contentHash: isNonEmptyString(contentHash) ? contentHash : schemaId, + canonicalSchemaJson: typeof canonicalSchemaJson === "string" ? canonicalSchemaJson : "", + compatibleSchemaIds, + members: members.map(decodeCatalogMember), + }; +} + +function decodeCatalogMember(value: unknown): CultCacheSchemaCatalogMember { + if (!Array.isArray(value)) { + throw new Error("CultCache schema catalog members must be MessagePack arrays."); + } + + const [ + slot = -1, + memberName = "", + typeName = "", + isReference = false, + isMany = false, + targetSchemaName = null, + isName = false, + indexAlias = null, + ] = value; + + if (!Number.isInteger(slot) || slot < 0 || !isNonEmptyString(memberName) || !isNonEmptyString(typeName)) { + throw new Error("CultCache schema catalog member has invalid slot, name, or type."); + } + + return { + slot, + memberName, + typeName, + isReference: isReference === true, + isMany: isMany === true, + targetSchemaName: typeof targetSchemaName === "string" ? targetSchemaName : null, + isName: isName === true, + indexAlias: typeof indexAlias === "string" ? indexAlias : null, + }; +} + +function decodeRecord(value: unknown): PersistedRecord { + if (!Array.isArray(value)) { + throw new Error("CultCache persisted records must be MessagePack arrays."); + } + + const [key = "", schemaId = "", storedAt = "", payload = new Uint8Array()] = value; + if (!isNonEmptyString(key) || !isNonEmptyString(schemaId) || !isNonEmptyString(storedAt)) { + throw new Error("CultCache persisted records must declare key, schemaId, and storedAt."); + } + + return { + key, + schemaId, + storedAt, + payload: normalizePayload(payload), + }; +} + +function decodeLegacyEnvelopeArray(decoded: unknown): Array<{ + key: string; + type: string; + payload: unknown; + storedAt: string; + schemaId?: string; +}> | undefined { + if (!Array.isArray(decoded)) { + return undefined; + } + + return legacyEnvelopeArraySchema.parse(decoded) as Array<{ + key: string; + type: string; + payload: unknown; + storedAt: string; + schemaId?: string; + }>; +} + function normalizePayload(payload: unknown): Uint8Array { if (payload instanceof Uint8Array) { return payload; @@ -161,3 +414,11 @@ function normalizePayload(payload: unknown): Uint8Array { function isObject(value: unknown): value is Record { return typeof value === "object" && value !== null; } + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +function compareOrdinal(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} diff --git a/src/types.ts b/src/types.ts index a3eb958..fe3795e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -18,6 +18,27 @@ export interface CultCacheDocumentFormatter { decode(payload: Uint8Array): TValue; } +export interface CultCacheSchemaCatalogMember { + slot: number; + memberName: string; + typeName: string; + isReference?: boolean; + isMany?: boolean; + targetSchemaName?: string | null; + isName?: boolean; + indexAlias?: string | null; +} + +export interface CultCacheSchemaCatalogEntry { + schemaId: string; + schemaName: string; + schemaVersion: string; + contentHash: string; + canonicalSchemaJson: string; + compatibleSchemaIds?: readonly string[]; + members?: readonly CultCacheSchemaCatalogMember[]; +} + export interface CultCacheDocumentIndexDefinition { name: string; accessor: CultCacheDocumentAccessor; @@ -26,7 +47,13 @@ export interface CultCacheDocumentIndexDefinition { export interface CultCacheDocumentDefinition { type: string; schema: TSchema; + schemaId?: string; schemaName?: string; + schemaVersion?: string; + contentHash?: string; + canonicalSchemaJson?: string; + compatibleSchemaIds?: readonly string[]; + members?: readonly CultCacheSchemaCatalogMember[]; global?: boolean; name?: CultCacheDocumentAccessor>; indexes?: @@ -52,6 +79,8 @@ export interface CultCacheEnvelope { type: string; payload: Uint8Array; storedAt: string; + schemaId?: string; + catalogEntry?: CultCacheSchemaCatalogEntry; } export interface PushAllOptions { diff --git a/test/cult-cache.test.ts b/test/cult-cache.test.ts index a006eab..dd5bba3 100644 --- a/test/cult-cache.test.ts +++ b/test/cult-cache.test.ts @@ -271,6 +271,118 @@ test("CultCache can ingest raw envelopes without re-encoding the payload", async assert.deepEqual(target.getRequiredEnvelope(noteDocument, "note:hello").payload, envelope.payload); }); +test("SingleFileMessagePackBackingStore writes the CultCache v1 snapshot shape", async () => { + const namedDocument = defineDocumentType({ + type: "tests.named_entry", + schema: z.object({ + Name: z.string(), + Value: z.string(), + }), + schemaId: "sha256:e7b97801b94190f3159012ede45b0069bb09ebf7920f7432c971bc86a0e08de8", + schemaName: "tests.named_entry", + schemaVersion: "tests.named_entry.v1", + contentHash: "sha256:23150930afcc1d84f0cb3012ccc2debcb9b4685f62083033bbaab0083f1e832e", + canonicalSchemaJson: "{\"schemaName\":\"tests.named_entry\",\"schemaVersion\":\"tests.named_entry.v1\",\"members\":[{\"slot\":0,\"name\":\"Name\",\"type\":\"System.String\",\"isReference\":false,\"many\":false,\"targetSchemaName\":null,\"indexAlias\":null,\"isName\":true},{\"slot\":1,\"name\":\"Value\",\"type\":\"System.String\",\"isReference\":false,\"many\":false,\"targetSchemaName\":null,\"indexAlias\":null,\"isName\":false}]}", + members: [ + { + slot: 0, + memberName: "Name", + typeName: "System.String", + isName: true, + }, + { + slot: 1, + memberName: "Value", + typeName: "System.String", + }, + ], + name: "Name", + }); + + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "snapshot.msgpack"); + const cache = CultCache.builder() + .withDocumentType(namedDocument) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + + await cache.put(namedDocument, "record-1", { + Name: "Teeth", + Value: "slot-array", + }); + + const snapshot = decode(await readFile(storePath)) as unknown[]; + assert.equal(snapshot[0], "cultcache.store.v1"); + + const catalog = snapshot[1] as unknown[][]; + assert.equal(catalog.length, 1); + assert.equal(catalog[0]?.[0], "sha256:e7b97801b94190f3159012ede45b0069bb09ebf7920f7432c971bc86a0e08de8"); + assert.equal(catalog[0]?.[1], "tests.named_entry"); + assert.equal(catalog[0]?.[2], "tests.named_entry.v1"); + + const records = snapshot[2] as unknown[][]; + assert.equal(records.length, 1); + assert.equal(records[0]?.[0], "record-1"); + assert.equal(records[0]?.[1], "sha256:e7b97801b94190f3159012ede45b0069bb09ebf7920f7432c971bc86a0e08de8"); + assert.ok(records[0]?.[3] instanceof Uint8Array); +}); + +test("SingleFileMessagePackBackingStore reads CultCache v1 snapshots by schema id", async () => { + const noteDocument = defineDocumentType({ + type: "tests.named_entry", + schema: z.object({ + Name: z.string(), + Value: z.string(), + }), + schemaId: "schema-1", + schemaName: "tests.named_entry", + schemaVersion: "tests.named_entry.v1", + contentHash: "hash-1", + canonicalSchemaJson: "{\"fields\":2}", + }); + + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "csharp.msgpack"); + await writeFile( + storePath, + encode([ + "cultcache.store.v1", + [ + [ + "schema-1", + "tests.named_entry", + "tests.named_entry.v1", + "hash-1", + "{\"fields\":2}", + ["schema-1"], + [], + ], + ], + [ + [ + "record-1", + "schema-1", + "2026-05-08T12:00:00Z", + encode({ + Name: "Teeth", + Value: "slot-array", + }), + ], + ], + ]), + ); + + const cache = CultCache.builder() + .withDocumentType(noteDocument) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + + await cache.pullAllBackingStores(); + assert.deepEqual(cache.getRequired(noteDocument, "record-1"), { + Name: "Teeth", + Value: "slot-array", + }); + assert.equal(cache.getRequiredEnvelope(noteDocument, "record-1").schemaId, "schema-1"); +}); + test("SingleFileMessagePackBackingStore heals legacy envelopes whose payload was persisted as an object", async () => { const noteDocument = defineDocumentType({ type: "note", @@ -306,7 +418,9 @@ test("SingleFileMessagePackBackingStore heals legacy envelopes whose payload was await cache.pullAllBackingStores(); assert.deepEqual(cache.getRequired(noteDocument, "note:hello"), envelopePayload); - const rewritten = decode(await readFile(storePath)) as Array<{ payload: unknown }>; - assert.equal(rewritten.length, 1); - assert.ok(rewritten[0]?.payload instanceof Uint8Array); + const rewritten = decode(await readFile(storePath)) as unknown[]; + assert.equal(rewritten[0], "cultcache.store.v1"); + const records = rewritten[2] as unknown[][]; + assert.equal(records.length, 1); + assert.ok(records[0]?.[3] instanceof Uint8Array); });