From 499887484c2bcfff8544b4a7f373c3accba97dae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 03:50:23 +0200 Subject: [PATCH 01/44] cultcache-ts: slot record and union codecs, held to the union-parity golden Unverified first pass; not yet compiled or run. Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/package.json | 2 +- packages/cultcache-ts/src/codec.ts | 229 +++++++++++ packages/cultcache-ts/src/cult-cache.ts | 5 +- packages/cultcache-ts/src/document.ts | 17 + packages/cultcache-ts/src/index.ts | 2 + packages/cultcache-ts/src/types.ts | 3 + .../cultcache-ts/test/union-parity.test.ts | 367 ++++++++++++++++++ 7 files changed, 622 insertions(+), 3 deletions(-) create mode 100644 packages/cultcache-ts/src/codec.ts create mode 100644 packages/cultcache-ts/test/union-parity.test.ts diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index c708bfc9..d31ab1b8 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -23,7 +23,7 @@ "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", - "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/cult-cache.test.js" + "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/cult-cache.test.js dist-test/test/union-parity.test.js" }, "keywords": [ "messagepack", diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts new file mode 100644 index 00000000..9e3a1552 --- /dev/null +++ b/packages/cultcache-ts/src/codec.ts @@ -0,0 +1,229 @@ +import { decode, encode } from "@msgpack/msgpack"; + +import type { CultCacheDocumentFormatter, CultCacheSchemaCatalogMember } from "./types"; + +// A codec turns a value into the MessagePack-ready shape the reference runtimes write and back. Slot records are +// positional arrays with nil gaps; a union value is { kind: armName, ...armFields }, written [armKey, [armSlots...]]. +// Refusals name the union or the slot and the key or code, and never echo a decoded value. +export interface Codec { + encode(value: TValue): unknown; + decode(raw: unknown): TValue; +} + +export interface SlotsCodec extends Codec { + readonly members: readonly CultCacheSchemaCatalogMember[]; + readonly names: readonly string[]; +} + +function expect(raw: unknown, what: string, ok: boolean): asserts ok { + if (!ok) { + throw new Error(`expected ${what}, found ${raw === null ? "nil" : Array.isArray(raw) ? "array" : typeof raw}`); + } +} + +export const string: Codec = { + encode: (value) => (expect(value, "a string", typeof value === "string"), value), + decode: (raw) => (expect(raw, "a string", typeof raw === "string"), raw as string), +}; + +export const number: Codec = { + encode: (value) => (expect(value, "a number", typeof value === "number" && Number.isFinite(value)), value), + decode: (raw) => (expect(raw, "a number", typeof raw === "number" && Number.isFinite(raw)), raw as number), +}; + +export const boolean: Codec = { + encode: (value) => (expect(value, "a boolean", typeof value === "boolean"), value), + decode: (raw) => (expect(raw, "a boolean", typeof raw === "boolean"), raw as boolean), +}; + +export const bytes: Codec = { + encode: (value) => (expect(value, "bytes", value instanceof Uint8Array), value), + decode: (raw) => (expect(raw, "bytes", raw instanceof Uint8Array), raw as Uint8Array), +}; + +export function list(item: Codec): Codec { + return { + encode(value) { + expect(value, "an array", Array.isArray(value)); + return value.map((element, index) => at(`element ${index}`, () => item.encode(element))); + }, + decode(raw) { + expect(raw, "an array", Array.isArray(raw)); + return (raw as unknown[]).map((element, index) => at(`element ${index}`, () => item.decode(element))); + }, + }; +} + +export function nullable(inner: Codec): Codec { + return { + encode: (value) => (value === null || value === undefined ? null : inner.encode(value)), + decode: (raw) => (raw === null ? null : inner.decode(raw)), + }; +} + +export function lazy(get: () => Codec): Codec { + let resolved: Codec | undefined; + const codec = () => (resolved ??= get()); + return { + encode: (value) => codec().encode(value), + decode: (raw) => codec().decode(raw), + }; +} + +// Prefixes a refusal with where it happened, so a nested refusal still names its field and its union. +function at(where: string, run: () => TResult): TResult { + try { + return run(); + } catch (error) { + throw new Error(`${where}: ${(error as Error).message}`); + } +} + +type SlotSpec = + | readonly [name: TName, codec: Codec] + | readonly [name: TName, codec: Codec, defaultValue: TValue | (() => TValue)]; + +type SlotsSpec = Readonly }[keyof TValue & string]>>; + +function slotKeys(spec: Readonly>, owner: string): number[] { + const keys = Object.keys(spec).map((text) => { + const key = Number(text); + if (!Number.isInteger(key) || key < 0 || String(key) !== text) { + throw new Error(`${owner} declares the key "${text}", which is not a non-negative integer.`); + } + return key; + }); + return keys.sort((left, right) => left - right); +} + +function pascalCase(name: string): string { + return name + .split(/[^A-Za-z0-9]+/u) + .filter((part) => part.length > 0) + .map((part) => part[0].toUpperCase() + part.slice(1)) + .join(""); +} + +// A positional record: slot `key` holds field `name`. Gaps are written nil and skipped on read, extra trailing slots +// are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that +// declares a default also takes the default (cultcache-py's rule). +export function slots(spec: SlotsSpec): SlotsCodec { + const keys = slotKeys(spec, "A slot record"); + if (keys.length === 0) { + throw new Error("A slot record declares no slots."); + } + const fields = keys.map((key) => { + const [name, codec, ...rest] = (spec as unknown as Readonly>>)[key]; + return { key, name, codec, hasDefault: rest.length > 0, defaultValue: rest[0] }; + }); + const seen = new Set(); + for (const field of fields) { + if (seen.has(field.name)) { + throw new Error(`A slot record declares the field "${field.name}" twice.`); + } + seen.add(field.name); + } + const length = keys[keys.length - 1] + 1; + const defaultOf = (field: (typeof fields)[number]): unknown => + typeof field.defaultValue === "function" ? (field.defaultValue as () => unknown)() : field.defaultValue; + + return { + names: fields.map((field) => field.name), + members: fields.map((field) => ({ slot: field.key, memberName: pascalCase(field.name), typeName: "System.Object" })), + encode(value) { + expect(value, "a record", typeof value === "object" && value !== null); + const out: unknown[] = new Array(length).fill(null); + for (const field of fields) { + out[field.key] = at(`${field.name} (slot ${field.key})`, () => + field.codec.encode((value as Record)[field.name])); + } + return out; + }, + decode(raw) { + expect(raw, "an array of slots", Array.isArray(raw)); + const input = raw as unknown[]; + const out: Record = {}; + for (const field of fields) { + const present = field.key < input.length && !(input[field.key] === null && field.hasDefault); + if (!present && !field.hasDefault) { + throw new Error(`${field.name} (slot ${field.key}): the slot is missing and declares no default.`); + } + out[field.name] = present + ? at(`${field.name} (slot ${field.key})`, () => field.codec.decode(input[field.key])) + : defaultOf(field); + } + return out as TValue; + }, + }; +} + +// An arm with no slots. It is written [key, []], never [key, nil]. +export const unit = Symbol("cultcache.unit-arm"); + +type ArmSpec = readonly [armName: TKind, body: SlotsCodec | typeof unit]; + +// A tagged union written [armKey, [armSlots...]]. The reader refuses an undeclared key, naming the union, the key and +// the declared keys; there is no fallback arm. +export function union( + name: string, + arms: Readonly>>, +): Codec { + const keys = slotKeys(arms, `Union ${name}`); + if (keys.length === 0) { + throw new Error(`Union ${name} declares no arms.`); + } + const byKey = new Map | typeof unit }>(); + const byName = new Map(); + for (const key of keys) { + const [armName, body] = arms[key]; + if (byName.has(armName)) { + throw new Error(`Union ${name} declares the arm "${armName}" twice.`); + } + if (body !== unit && body.names.includes("kind")) { + throw new Error(`Union ${name} arm "${armName}" declares a slot named kind, which is the union's own tag.`); + } + byKey.set(key, { armName, body }); + byName.set(armName, key); + } + const declared = keys.join(", "); + + return { + encode(value) { + if (value === null || value === undefined) { + return null; + } + const key = byName.get((value as { kind?: string }).kind as string); + if (key === undefined) { + throw new Error(`Union ${name}: the value's kind is not one of its arms (${[...byName.keys()].join(", ")}).`); + } + const { armName, body } = byKey.get(key)!; + return [key, body === unit ? [] : at(`arm ${armName}`, () => body.encode(value))]; + }, + decode(raw) { + if (raw === null) { + return null; + } + if (!Array.isArray(raw) || raw.length !== 2 || !Array.isArray(raw[1])) { + throw new Error(`Union ${name}: expected [key, slots] and found something else.`); + } + const key: unknown = raw[0]; + if (typeof key !== "number" || !Number.isInteger(key)) { + throw new Error(`Union ${name}: the arm key is not an integer (declared keys: ${declared}).`); + } + const arm = byKey.get(key); + if (arm === undefined) { + throw new Error(`Union ${name}: unknown arm key ${key} (declared keys: ${declared}).`); + } + const fields = arm.body === unit ? {} : at(`arm ${arm.armName}`, () => (arm.body as SlotsCodec).decode(raw[1])); + return { kind: arm.armName, ...fields } as unknown as TValue; + }, + }; +} + +// The formatter a document gets from a slot codec: the codec's value as one MessagePack payload. +export function codecFormatter(codec: SlotsCodec): CultCacheDocumentFormatter { + return { + encode: (value) => encode(codec.encode(value)), + decode: (payload) => codec.decode(decode(payload)), + }; +} diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 57766be9..a1c0a3d4 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,5 +1,6 @@ import { decode, encode } from "@msgpack/msgpack"; +import { codecFormatter } from "./codec"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, @@ -676,10 +677,10 @@ export class CultCache { } #createFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { - const userFormatter = definition.formatter ?? { + const userFormatter = definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : { encode: (value: unknown) => encode(value), decode: (payload: Uint8Array) => decode(payload), - }; + }); return { encode: (value: unknown) => { diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index ff85d5af..70cdd7b2 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -66,6 +66,23 @@ export function defineDocumentType( } } + if (definition.codec) { + const codec = definition.codec; + if (definition.formatter) { + throw new Error(`CultCache document type "${definition.type}" declares both a codec and a formatter.`); + } + const indexAlias = (slotName: string) => + !Array.isArray(definition.indexes) && definition.indexes + ? Object.entries(definition.indexes).find(([, accessor]) => accessor === slotName)?.[0] ?? null + : null; + const members = definition.members ?? codec.members.map((member, index) => ({ + ...member, + isName: definition.name === codec.names[index], + indexAlias: indexAlias(codec.names[index]), + })); + return Object.freeze({ ...definition, members }); + } + return Object.freeze({ ...definition }); } diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 92f7b481..78e9c7c0 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -1,3 +1,5 @@ +export * as codecs from "./codec"; +export type { Codec, SlotsCodec } from "./codec"; export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; export { defineDocumentRegistry, defineDocumentType } from "./document"; diff --git a/packages/cultcache-ts/src/types.ts b/packages/cultcache-ts/src/types.ts index fe3795e1..dada7fc0 100644 --- a/packages/cultcache-ts/src/types.ts +++ b/packages/cultcache-ts/src/types.ts @@ -1,3 +1,5 @@ +import type { SlotsCodec } from "./codec"; + export type CultCacheIndexScalar = string | number | boolean | bigint; export interface CultCacheSchema { @@ -60,6 +62,7 @@ export interface CultCacheDocumentDefinition>>> | readonly CultCacheDocumentIndexDefinition>[]; formatter?: CultCacheDocumentFormatter>; + codec?: SlotsCodec & object>; } export type CultCacheDocumentValue = InferCultCacheSchemaValue< diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts new file mode 100644 index 00000000..801cff93 --- /dev/null +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -0,0 +1,367 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { test } from "node:test"; + +import { decode, encode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { codecs } from "../src"; +import { CultCache } from "../src/cult-cache"; +import { defineDocumentType } from "../src/document"; +import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; +import type { CacheBackingStore, CultCacheEnvelope } from "../src/types"; + +const { boolean, bytes, lazy, list, nullable, number, slots, string, union, unit } = codecs; + +// The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected +// outcome is read from those committed files; only the value the payload must decode to is spelled here. +const vectors = resolve(__dirname, "../../../../tests/vectors/union-parity"); +const read = async (name: string) => new Uint8Array(await readFile(join(vectors, name))); + +// The string the C# writer put inside every refusal payload, to prove a refusal never echoes the payload. +const CANARY = "CANARY-union-golden-4c2a"; + +type Node = + | { kind: "Box"; id: string; children: (Node | null)[] } + | { kind: "Text"; id: string; content: string } + | { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] } + | { kind: "Image"; id: string; uri: string; alt: string }; +interface Composition { name: string; body: Node | null } +type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" }; +type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; +type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; +interface Surface { + id: string; + version: number; + root: Node | null; + outcomes: (Outcome | null)[]; + part: Item | null; + crafted: CraftedItem | null; + missing: Node | null; +} + +const composition = slots({ 0: ["name", string], 1: ["body", lazy(() => node)] }); +const node = union("Node", { + 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })], + 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })], + 2: ["Use", slots<{ id: string; composition: Composition; slots: (Node | null)[] }>({ + 0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))], + })], + 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], +}); +const outcome = union("Outcome", { + 0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })], + 1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", number] })], + 2: ["Denied", unit], +}); +const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", number] }); +const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", number] }); +const item = union("Item", { + 0: ["Gem", slots<{ name: string }>({ 0: ["name", string] })], + 1: ["Tool", toolArm], + 2: ["Part", partArm], +}); +const craftedItem = union("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }); +const surfaceCodec = slots({ + 0: ["id", string], + 1: ["version", number], + 2: ["root", node], + 3: ["outcomes", list(outcome)], + 4: ["part", item], + 5: ["crafted", craftedItem], + 6: ["missing", node, null], +}); + +const nodeSchema: z.ZodType = z.lazy(() => + z.union([ + z.null(), + z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Box"), id: z.string(), children: z.array(nodeSchema) }), + z.object({ kind: z.literal("Text"), id: z.string(), content: z.string() }), + z.object({ + kind: z.literal("Use"), + id: z.string(), + composition: z.object({ name: z.string(), body: nodeSchema }), + slots: z.array(nodeSchema), + }), + z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }), + ]), + ])); +const surfaceSchema = z.object({ + id: z.string(), + version: z.number(), + root: nodeSchema, + outcomes: z.array(z.union([ + z.null(), + z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Pending"), message: z.string() }), + z.object({ kind: z.literal("Completed"), sourceVersion: z.number() }), + z.object({ kind: z.literal("Denied") }), + ]), + ])), + part: z.union([z.null(), z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Gem"), name: z.string() }), + z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }), + z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }), + ])]), + crafted: z.union([z.null(), z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }), + z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }), + ])]), + missing: nodeSchema, +}) as unknown as z.ZodType; + +function surfaceDocument(schemaId?: string) { + return defineDocumentType({ + type: "tests.union_surface", + schemaId: schemaId ?? "tests.union_surface.v1", + schemaName: "tests.union_surface", + schemaVersion: "tests.union_surface.v1", + schema: surfaceSchema, + codec: surfaceCodec, + name: "id", + }); +} + +const golden: Surface = { + id: "surface", + version: 3, + root: { + kind: "Box", + id: "root", + children: [ + { kind: "Text", id: "t1", content: "hello" }, + { + kind: "Use", + id: "u1", + composition: { + name: "comp", + body: { + kind: "Box", + id: "inner", + children: [ + { kind: "Text", id: "deep", content: "x" }, + { kind: "Image", id: "img", uri: "uri://a", alt: "alt" }, + ], + }, + }, + slots: [{ kind: "Text", id: "s1", content: "slot" }, null, { kind: "Box", id: "b2", children: [] }], + }, + { kind: "Image", id: "i2", uri: "uri://b", alt: "" }, + ], + }, + outcomes: [{ kind: "Pending", message: "wait" }, { kind: "Completed", sourceVersion: 7 }, { kind: "Denied" }], + part: { kind: "Tool", name: "hammer", uses: 12 }, + crafted: { kind: "Part", name: "gear", count: 4 }, + missing: null, +}; + +const hex = (data: Uint8Array) => Buffer.from(data).toString("hex"); + +async function outcomeEntries() { + const entries = decode(await read("outcomes.msgpack")) as [string, string, string, number | null, string][]; + return entries.map(([file, outcome, unionName, key, detail]) => ({ file, outcome, unionName, key, detail })); +} + +test("the union surface vector decodes to the constructed value and re-encodes byte-identical", async () => { + const committed = await read("union-surface.msgpack"); + const decoded = surfaceCodec.decode(decode(committed)); + assert.deepEqual(decoded, golden); + assert.equal(hex(encode(surfaceCodec.encode(decoded))), hex(committed)); + assert.equal(hex(encode(surfaceCodec.encode(golden))), hex(committed)); +}); + +test("a unit arm is written [key, []] and its payload must be an array", () => { + assert.deepEqual(outcome.encode({ kind: "Denied" }), [2, []]); + assert.throws(() => outcome.decode([2, null]), /Union Outcome/u); +}); + +test("union-surface.cc reads through SingleFileMessagePackBackingStore and writes the same payload", async () => { + const dir = await mkdtemp(join(tmpdir(), "union-parity-")); + const file = join(dir, "union-surface.cc"); + await writeFile(file, await read("union-surface.cc")); + const [envelope] = await new SingleFileMessagePackBackingStore(file).pullAll(); + assert.equal(hex(envelope.payload), hex(await read("union-surface.msgpack"))); + + const document = surfaceDocument(envelope.schemaId); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + await cache.pullAllBackingStores(); + assert.deepEqual(cache.get(document, envelope.key), golden); + + const written = join(dir, "written.cc"); + const fresh = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(written)).build(); + await fresh.put(document, envelope.key, golden); + const [rewritten] = await new SingleFileMessagePackBackingStore(written).pullAll(); + assert.equal(hex(rewritten.payload), hex(envelope.payload)); +}); + +test("every outcome vector is listed and holds: refusals name the union and key, never the payload", async () => { + const entries = await outcomeEntries(); + assert.equal(entries.length, 9); + const document = surfaceDocument(); + let refused = 0; + let readOnly = 0; + + for (const { file, outcome: expected, unionName, key, detail } of entries) { + const payload = await read(file); + if (expected === "refused") { + refused += 1; + assert.ok(Buffer.from(payload).includes(CANARY) || detail === "arity", `${file} carries the canary`); + assert.throws( + () => surfaceCodec.decode(decode(payload)), + (error: Error) => { + assert.ok(!error.message.includes(CANARY), `${file}: the refusal echoed the payload`); + if (detail === "arity") { + return error.message.includes(`Union ${unionName}`) && error.message.includes("expected [key, slots]"); + } + return error.message.includes(`Union ${unionName}:`) && error.message.includes(`key ${key} `); + }, + file, + ); + const refusingCache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(payload)])).build(); + await assert.rejects(() => refusingCache.pullAllBackingStores(), new RegExp(`Union ${unionName}`, "u"), file); + } else { + readOnly += 1; + assert.equal(expected, "read-only"); + assert.deepEqual(surfaceCodec.decode(decode(payload)), golden, `${file}: the extra slot is read past`); + } + } + assert.deepEqual([refused, readOnly], [5, 4]); +}); + +test("a refusal lists the declared keys, so a key only the outer union declares is unknown to the inner", async () => { + const entry = (await outcomeEntries()).find((candidate) => candidate.file === "inner-union-outer-key.msgpack")!; + const payload = decode(await read(entry.file)) as unknown[]; + assert.throws( + () => surfaceCodec.decode(payload), + /Union CraftedItem: unknown arm key 2 \(declared keys: 0, 1\)\./u, + ); +}); + +test("an arm that is not [key, slots] is refused naming the union, and a non-integer key is not echoed", () => { + for (const bad of [[1], [1, null], [1, [], []], "x", 7, {}]) { + assert.throws(() => node.decode(bad), /^Error: Union Node: expected \[key, slots\]/u); + } + assert.throws( + () => node.decode([CANARY, []]), + (error: Error) => error.message.includes("Union Node") && error.message.includes("not an integer") && !error.message.includes(CANARY), + ); +}); + +test("an arm key that is not a declared key is refused, with no fallback", () => { + assert.throws(() => node.decode([4, []]), /Union Node: unknown arm key 4 \(declared keys: 0, 1, 2, 5\)/u); + assert.throws(() => outcome.decode([3, []]), /Union Outcome: unknown arm key 3/u); +}); + +test("a union encodes by kind and refuses a kind it does not declare without echoing it", () => { + assert.deepEqual(node.encode({ kind: "Text", id: "a", content: "b" }), [1, ["a", "b"]]); + assert.equal(node.encode(null), null); + assert.throws( + () => node.encode({ kind: CANARY, id: "a" } as unknown as Node), + (error: Error) => error.message.includes("Union Node") && !error.message.includes(CANARY), + ); +}); + +test("slots: a gap writes nil, a missing slot reads its default, one without a default is refused naming the slot", () => { + const record = slots<{ a: string; c: string; d: number }>({ 0: ["a", string], 2: ["c", string], 4: ["d", number, 9] }); + assert.deepEqual(record.encode({ a: "x", c: "y", d: 1 }), ["x", null, "y", null, 1]); + assert.deepEqual(record.decode(["x", null, "y"]), { a: "x", c: "y", d: 9 }); + assert.deepEqual(record.decode(["x", 5, "y", null, null]), { a: "x", c: "y", d: 9 }); + assert.throws(() => record.decode(["x"]), /c \(slot 2\): the slot is missing and declares no default/u); + assert.deepEqual(record.decode(["x", null, "y", null, 1, "extra", "slots"]), { a: "x", c: "y", d: 1 }); + assert.throws(() => record.decode({ a: "x" }), /expected an array of slots/u); +}); + +test("slots: a default may be a function, and nil in a defaulted slot takes it", () => { + const record = slots<{ tags: string[] }>({ 0: ["tags", list(string), () => ["fresh"]] }); + assert.deepEqual(record.decode([]), { tags: ["fresh"] }); + assert.deepEqual(record.decode([null]), { tags: ["fresh"] }); +}); + +test("a slot refusal names the field and slot and never the value", () => { + const record = slots<{ name: string }>({ 3: ["name", string] }); + assert.throws( + () => record.decode([null, null, null, { secret: CANARY }]), + (error: Error) => error.message.startsWith("name (slot 3):") && !error.message.includes(CANARY), + ); +}); + +test("leaf codecs hold their types", () => { + assert.throws(() => number.encode(Number.NaN), /expected a number/u); + assert.throws(() => string.decode(5), /expected a string, found number/u); + assert.throws(() => boolean.decode("true"), /expected a boolean/u); + assert.equal(boolean.decode(true), true); +}); + +test("definitions are refused at definition time", () => { + assert.throws(() => union<{ kind: "A" }>("U", { 0: ["A", slots<{ kind: string }>({ 0: ["kind", string] })] }), /Union U arm "A" declares a slot named kind/u); + assert.throws(() => union<{ kind: "A" | "B" }>("U", { 0: ["A", unit], 1: ["A" as "B", unit] }), /declares the arm "A" twice/u); + assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /not a non-negative integer/u); + assert.throws(() => union<{ kind: "A" }>("U", {} as never), /declares no arms/u); + assert.throws(() => slots<{ a: string; b: string }>({ 0: ["a", string], 1: ["a" as "b", string] }), /declares the field "a" twice/u); + assert.throws(() => slots<{ a: string }>({}), /declares no slots/u); + assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), /not a non-negative integer/u); + assert.throws( + () => defineDocumentType({ + type: "t", + schema: surfaceSchema, + codec: surfaceCodec, + formatter: { encode: () => new Uint8Array(), decode: () => golden }, + }), + /declares both a codec and a formatter/u, + ); +}); + +test("a codec supplies the document's catalog members unless it is given its own", () => { + const document = defineDocumentType({ + type: "t", + schema: z.object({ a: z.string(), the_b: z.string() }), + codec: slots<{ a: string; the_b: string }>({ 0: ["a", string], 3: ["the_b", string] }), + name: "a", + indexes: { byB: "the_b" }, + }); + assert.deepEqual(document.members, [ + { slot: 0, memberName: "A", typeName: "System.Object", isName: true, indexAlias: null }, + { slot: 3, memberName: "TheB", typeName: "System.Object", isName: false, indexAlias: "byB" }, + ]); + const given = [{ slot: 9, memberName: "Mine", typeName: "System.String" }]; + const own = defineDocumentType({ type: "t", schema: z.object({ a: z.string() }), codec: slots<{ a: string }>({ 0: ["a", string] }), members: given }); + assert.deepEqual(own.members, given); +}); + +test("schema.parse still runs on a decoded value, and a document without a codec still writes a map", async () => { + const strict = defineDocumentType({ + type: "t", + schema: z.object({ a: z.string().min(3) }), + codec: slots<{ a: string }>({ 0: ["a", string] }), + }); + const store = memoryStore([{ key: "k", type: "t", payload: encode(["xy"]), storedAt: new Date().toISOString() }]); + await assert.rejects(() => CultCache.builder().withDocumentType(strict).withGenericStore(store).build().pullAllBackingStores()); + + const plain = defineDocumentType({ type: "p", schema: z.object({ id: z.string() }) }); + const holder = memoryStore([]); + await CultCache.builder().withDocumentType(plain).withGenericStore(holder).build().put(plain, "k", { id: "d" }); + assert.deepEqual(decode(holder.entries[0].payload), { id: "d" }); +}); + +test("bytes and nullable round-trip through a slot record", () => { + const record = slots<{ blob: Uint8Array; maybe: string | null }>({ 0: ["blob", bytes], 1: ["maybe", nullable(string)] }); + const wire = encode(record.encode({ blob: new Uint8Array([1, 2]), maybe: null })); + assert.deepEqual(record.decode(decode(wire)), { blob: new Uint8Array([1, 2]), maybe: null }); + assert.throws(() => record.decode([new Uint8Array(), 4]), /maybe \(slot 1\): expected a string/u); +}); + +function envelopeOf(payload: Uint8Array): CultCacheEnvelope { + return { key: "surface", type: "tests.union_surface", payload, storedAt: new Date().toISOString() }; +} + +function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } { + return { + entries, + async pullAll() { return entries.map((entry) => ({ ...entry })); }, + async push(entry) { entries.push(entry); }, + async delete() {}, + }; +} From f43a0d26f9e32cdef8fbd0c38aaa8c505e414073 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 03:51:11 +0200 Subject: [PATCH 02/44] cultcache-ts codec: unit arm is a frozen object, recursive test codec annotated Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 13 ++++++++----- packages/cultcache-ts/test/union-parity.test.ts | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 9e3a1552..44138bc9 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -158,9 +158,12 @@ export function slots(spec: SlotsSpec): SlotsCode } // An arm with no slots. It is written [key, []], never [key, nil]. -export const unit = Symbol("cultcache.unit-arm"); +export interface UnitArm { + readonly unit: true; +} +export const unit: UnitArm = Object.freeze({ unit: true }); -type ArmSpec = readonly [armName: TKind, body: SlotsCodec | typeof unit]; +type ArmSpec = readonly [armName: TKind, body: SlotsCodec | UnitArm]; // A tagged union written [armKey, [armSlots...]]. The reader refuses an undeclared key, naming the union, the key and // the declared keys; there is no fallback arm. @@ -172,14 +175,14 @@ export function union( if (keys.length === 0) { throw new Error(`Union ${name} declares no arms.`); } - const byKey = new Map | typeof unit }>(); + const byKey = new Map | UnitArm }>(); const byName = new Map(); for (const key of keys) { const [armName, body] = arms[key]; if (byName.has(armName)) { throw new Error(`Union ${name} declares the arm "${armName}" twice.`); } - if (body !== unit && body.names.includes("kind")) { + if (body !== unit && (body as SlotsCodec).names.includes("kind")) { throw new Error(`Union ${name} arm "${armName}" declares a slot named kind, which is the union's own tag.`); } byKey.set(key, { armName, body }); @@ -197,7 +200,7 @@ export function union( throw new Error(`Union ${name}: the value's kind is not one of its arms (${[...byName.keys()].join(", ")}).`); } const { armName, body } = byKey.get(key)!; - return [key, body === unit ? [] : at(`arm ${armName}`, () => body.encode(value))]; + return [key, body === unit ? [] : at(`arm ${armName}`, () => (body as SlotsCodec).encode(value))]; }, decode(raw) { if (raw === null) { diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 801cff93..560b7273 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -8,6 +8,7 @@ import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; import { codecs } from "../src"; +import type { Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; @@ -43,7 +44,7 @@ interface Surface { } const composition = slots({ 0: ["name", string], 1: ["body", lazy(() => node)] }); -const node = union("Node", { +const node: Codec = union("Node", { 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })], 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })], 2: ["Use", slots<{ id: string; composition: Composition; slots: (Node | null)[] }>({ From 82aab7d83ead55ffa425804cfb42b73d62e4ce22 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 03:51:37 +0200 Subject: [PATCH 03/44] union-parity test: the canary check counts the payloads that carry it Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/test/union-parity.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 560b7273..a8e2e1ba 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -203,13 +203,14 @@ test("every outcome vector is listed and holds: refusals name the union and key, assert.equal(entries.length, 9); const document = surfaceDocument(); let refused = 0; + const carrying: string[] = []; let readOnly = 0; for (const { file, outcome: expected, unionName, key, detail } of entries) { const payload = await read(file); if (expected === "refused") { refused += 1; - assert.ok(Buffer.from(payload).includes(CANARY) || detail === "arity", `${file} carries the canary`); + if (Buffer.from(payload).includes(CANARY)) carrying.push(file); assert.throws( () => surfaceCodec.decode(decode(payload)), (error: Error) => { @@ -230,6 +231,7 @@ test("every outcome vector is listed and holds: refusals name the union and key, } } assert.deepEqual([refused, readOnly], [5, 4]); + assert.deepEqual(carrying, ["unknown-arm-field.msgpack", "unknown-arm-list.msgpack", "unknown-arm-nested.msgpack"], "the canary is in the payloads whose arm is unknown"); }); test("a refusal lists the declared keys, so a key only the outer union declares is unknown to the inner", async () => { From 21dec1ef56d2cef7d61d290717f849de6a2412e4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 03:54:30 +0200 Subject: [PATCH 04/44] union-parity test: find the vectors by walking up, so the test runs from source as well as from dist-test Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/test/union-parity.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index a8e2e1ba..b64381a8 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -18,7 +19,13 @@ const { boolean, bytes, lazy, list, nullable, number, slots, string, union, unit // The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected // outcome is read from those committed files; only the value the payload must decode to is spelled here. -const vectors = resolve(__dirname, "../../../../tests/vectors/union-parity"); +function findVectors(from: string): string { + for (let dir = from; ; dir = resolve(dir, "..")) { + if (existsSync(join(dir, "tests", "vectors", "union-parity"))) return join(dir, "tests", "vectors", "union-parity"); + if (resolve(dir, "..") === dir) throw new Error("tests/vectors/union-parity not found above the test."); + } +} +const vectors = findVectors(__dirname); const read = async (name: string) => new Uint8Array(await readFile(join(vectors, name))); // The string the C# writer put inside every refusal payload, to prove a refusal never echoes the payload. From 344630259a1ac938ffc6f0baced903d3f8258da6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:03:04 +0200 Subject: [PATCH 05/44] cultcache-ts codec: tests for refusal messages and leaf shapes; drop a redundant key sort and regex quantifier Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 6 +- .../cultcache-ts/test/union-parity.test.ts | 67 ++++++++++++++++--- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 44138bc9..c37b6a07 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -86,19 +86,19 @@ type SlotSpec = type SlotsSpec = Readonly }[keyof TValue & string]>>; function slotKeys(spec: Readonly>, owner: string): number[] { - const keys = Object.keys(spec).map((text) => { + // Object.keys lists integer keys in ascending order, so the declared keys come back sorted. + return Object.keys(spec).map((text) => { const key = Number(text); if (!Number.isInteger(key) || key < 0 || String(key) !== text) { throw new Error(`${owner} declares the key "${text}", which is not a non-negative integer.`); } return key; }); - return keys.sort((left, right) => left - right); } function pascalCase(name: string): string { return name - .split(/[^A-Za-z0-9]+/u) + .split(/[^A-Za-z0-9]/u) .filter((part) => part.length > 0) .map((part) => part[0].toUpperCase() + part.slice(1)) .join(""); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index b64381a8..41bbb0b0 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -298,21 +298,72 @@ test("a slot refusal names the field and slot and never the value", () => { ); }); -test("leaf codecs hold their types", () => { - assert.throws(() => number.encode(Number.NaN), /expected a number/u); - assert.throws(() => string.decode(5), /expected a string, found number/u); - assert.throws(() => boolean.decode("true"), /expected a boolean/u); - assert.equal(boolean.decode(true), true); +test("leaf codecs refuse the wrong shape on both sides, saying what was expected and what kind was found", () => { + const table: [string, Codec, string, unknown, string][] = [ + ["string", string, "a string", 5, "number"], + ["number", number, "a number", "5", "string"], + ["number", number, "a number", Number.POSITIVE_INFINITY, "number"], + ["number", number, "a number", Number.NaN, "number"], + ["boolean", boolean, "a boolean", "true", "string"], + ["bytes", bytes, "bytes", [1, 2], "array"], + ["string", string, "a string", null, "nil"], + ["list", list(string), "an array", "abc", "string"], + ]; + for (const [label, codec, what, bad, found] of table) { + for (const side of ["encode", "decode"] as const) { + assert.throws(() => codec[side](bad), new RegExp(`^Error: expected ${what}, found ${found}$`, "u"), `${label} ${side}`); + } + } + for (const [codec, good] of [[string, "s"], [number, 1.5], [boolean, false], [bytes, new Uint8Array([7])], [list(number), [1, 2]]] as [Codec, unknown][]) { + assert.deepEqual(codec.decode(codec.encode(good)), good); + } +}); + +test("a list element refusal names the element, and nullable writes nil for null and undefined", () => { + const numbers = list(number); + assert.throws(() => numbers.encode([1, "x"]), /^Error: element 1: expected a number, found string$/u); + assert.throws(() => numbers.decode([1, 2, null]), /^Error: element 2: expected a number, found nil$/u); + const maybe = nullable(number); + assert.equal(maybe.encode(null), null); + assert.equal(maybe.encode(undefined as unknown as null), null); + assert.equal(maybe.encode(0), 0); + assert.equal(maybe.decode(null), null); + assert.equal(maybe.decode(0), 0); + assert.throws(() => maybe.decode("x"), /expected a number/u); +}); + +test("a record refuses what is not a record and names the field and slot of a bad value", () => { + const record = slots<{ name: string }>({ 3: ["name", string] }); + assert.throws(() => record.encode(null as unknown as { name: string }), /^Error: expected a record, found nil$/u); + assert.throws(() => record.encode("x" as unknown as { name: string }), /^Error: expected a record, found string$/u); + assert.throws(() => record.encode({ name: 5 as unknown as string }), /^Error: name \(slot 3\): expected a string, found number$/u); +}); + +test("a union names itself, its arm and the field when an arm's slots are refused", () => { + assert.throws(() => node.encode({ kind: "Text", id: "a", content: 5 as unknown as string }), /^Error: arm Text: content \(slot 1\): expected a string, found number$/u); + assert.throws(() => node.decode([1, ["a", 5]]), /^Error: arm Text: content \(slot 1\): expected a string, found number$/u); + assert.equal(node.encode(undefined as unknown as null), null); + assert.throws( + () => node.encode({ kind: "Nope" } as unknown as Node), + /^Error: Union Node: the value's kind is not one of its arms \(Box, Text, Use, Image\)\.$/u, + ); +}); + +test("slot names become catalog member names in Pascal case", () => { + const record = slots<{ _hidden: string; two_words: string; Mixed9: string }>({ 0: ["_hidden", string], 1: ["two_words", string], 2: ["Mixed9", string] }); + assert.deepEqual(record.members.map((member) => member.memberName), ["Hidden", "TwoWords", "Mixed9"]); }); test("definitions are refused at definition time", () => { assert.throws(() => union<{ kind: "A" }>("U", { 0: ["A", slots<{ kind: string }>({ 0: ["kind", string] })] }), /Union U arm "A" declares a slot named kind/u); assert.throws(() => union<{ kind: "A" | "B" }>("U", { 0: ["A", unit], 1: ["A" as "B", unit] }), /declares the arm "A" twice/u); - assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /not a non-negative integer/u); - assert.throws(() => union<{ kind: "A" }>("U", {} as never), /declares no arms/u); + assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not a non-negative integer\.$/u); + assert.throws(() => union<{ kind: "A" }>("U", { "01": ["A", unit] } as never), /key "01"/u); + assert.throws(() => union<{ kind: "A" }>("U", { "1.5": ["A", unit] } as never), /key "1\.5"/u); + assert.throws(() => union<{ kind: "A" }>("U", {} as never), /^Error: Union U declares no arms\.$/u); assert.throws(() => slots<{ a: string; b: string }>({ 0: ["a", string], 1: ["a" as "b", string] }), /declares the field "a" twice/u); assert.throws(() => slots<{ a: string }>({}), /declares no slots/u); - assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), /not a non-negative integer/u); + assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), /^Error: A slot record declares the key "-1"/u); assert.throws( () => defineDocumentType({ type: "t", From 9073d582e7fdcf05db43d77e9b130efdbaa1ee30 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:06:43 +0200 Subject: [PATCH 06/44] cultcache-ts codec: Number.isFinite alone decides a number Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index c37b6a07..7381c348 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -27,8 +27,8 @@ export const string: Codec = { }; export const number: Codec = { - encode: (value) => (expect(value, "a number", typeof value === "number" && Number.isFinite(value)), value), - decode: (raw) => (expect(raw, "a number", typeof raw === "number" && Number.isFinite(raw)), raw as number), + encode: (value) => (expect(value, "a number", Number.isFinite(value)), value), + decode: (raw) => (expect(raw, "a number", Number.isFinite(raw)), raw as number), }; export const boolean: Codec = { From bea5564b3aa6e33d8ae8df880e6f3f7982810e22 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:09:57 +0200 Subject: [PATCH 07/44] union-parity test: the one list-element probe is cast so the test project compiles Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/test/union-parity.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 41bbb0b0..8aca877d 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -321,7 +321,7 @@ test("leaf codecs refuse the wrong shape on both sides, saying what was expected test("a list element refusal names the element, and nullable writes nil for null and undefined", () => { const numbers = list(number); - assert.throws(() => numbers.encode([1, "x"]), /^Error: element 1: expected a number, found string$/u); + assert.throws(() => numbers.encode([1, "x" as unknown as number]), /^Error: element 1: expected a number, found string$/u); assert.throws(() => numbers.decode([1, 2, null]), /^Error: element 2: expected a number, found nil$/u); const maybe = nullable(number); assert.equal(maybe.encode(null), null); From 5aca6a8902631f5a0005ac4fbb9cd3a58302f8f8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:13:56 +0200 Subject: [PATCH 08/44] cultcache-ts README: slot records and unions Docs only; the code and tests are unchanged from the previous commit. Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/README.md | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index d1730958..26b94ecc 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -234,6 +234,42 @@ The important boundary is this: So callers get typed values, and the persistence layer does not become an open polymorphic sewer. +## Slot Records and Unions + +`codec` writes the reference runtimes' record shape: a MessagePack array indexed by slot key, nil in a gap, and a +union as `[armKey, [armSlots...]]`. A union value is `{ kind: armName, ...armFields }`. + +```ts +import { codecs, defineDocumentType } from "@gamecult/cultcache-ts"; +const { slots, union, list, lazy, string } = codecs; + +type Node = + | { kind: "Box"; id: string; children: (Node | null)[] } + | { kind: "Text"; id: string; content: string } + | { kind: "Image"; id: string; uri: string; alt: string }; + +const node: codecs.Codec = union("Node", { + 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })], + 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })], + 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], // slot 2 retired +}); + +const sceneDocument = defineDocumentType({ + type: "scene", + schema: sceneSchema, + codec: slots({ 0: ["id", string], 1: ["root", node], 2: ["title", string, ""] }), + name: "id", +}); +``` + +- A slot is `[name, codec, default?]`. A gap is written nil and skipped on read; an extra trailing slot is skipped; a + slot missing from the payload (or nil, when a default is declared) takes its default, and is refused without one. +- A union refuses an undeclared arm key on read, naming the union, the key and the declared keys. Nothing falls back + to a default arm. An arm with no slots is `codecs.unit` and is written `[key, []]`. +- A refusal names the union, the field and the slot; it never echoes the value it refused. +- `codec` supplies the document's formatter and, unless `members` is given, its catalog members. A definition may not + carry both `codec` and `formatter`. `schema.parse` still runs on every decoded value. + ## Raw Envelope Fast Path For bit-compatible neighbors such as CultNet peers that already share the same From 485b570cf5a5b08da3d416168936c06748e9e635 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:49:13 +0200 Subject: [PATCH 09/44] cultcache-ts codec: reference-typed numeric leaves, a wire reader and writer, one owner for codec-vs-formatter (work in progress, tests not yet run) The codec path owns a small MessagePack reader and writer so it can tell an integer from a float and keep 64-bit integers whole. Leaves int32, int64, float32, float64 replace number; every codec carries the reference's type name and a slot record derives its catalog members from them. Not built or run at this commit. Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 148 +++++++++++++++--- packages/cultcache-ts/src/cult-cache.ts | 13 +- packages/cultcache-ts/src/document.ts | 53 +++++-- packages/cultcache-ts/src/wire.ts | 130 +++++++++++++++ .../cultcache-ts/test/union-parity.test.ts | 70 +++++---- 5 files changed, 337 insertions(+), 77 deletions(-) create mode 100644 packages/cultcache-ts/src/wire.ts diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 7381c348..508fb9d6 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -1,48 +1,117 @@ -import { decode, encode } from "@msgpack/msgpack"; - import type { CultCacheDocumentFormatter, CultCacheSchemaCatalogMember } from "./types"; +import { readWire, WireFloat, writeWire } from "./wire"; -// A codec turns a value into the MessagePack-ready shape the reference runtimes write and back. Slot records are -// positional arrays with nil gaps; a union value is { kind: armName, ...armFields }, written [armKey, [armSlots...]]. -// Refusals name the union or the slot and the key or code, and never echo a decoded value. +// A codec turns a value into the shape the reference runtimes write and back. Slot records are positional arrays with +// nil gaps; a union value is { kind: armName, ...armFields }, written [armKey, [armSlots...]]. Refusals name the union +// or the slot and the key or code, and never echo a decoded value. `typeName` is the reference's name for the type +// (CultSchemaTypeNames.FromType): a store's schema catalog carries it per slot, and the reference refuses a store whose +// slot names another type. The leaves know theirs; a union or a record the codec cannot name from C# is given it. export interface Codec { encode(value: TValue): unknown; decode(raw: unknown): TValue; + readonly typeName?: string; + // True for a type the reference makes Nullable when the slot may be nil (int, long, float, double, bool). + readonly valueType?: boolean; } export interface SlotsCodec extends Codec { + // Throws when a slot's codec has no typeName: the catalog cannot be derived without the reference's names. readonly members: readonly CultCacheSchemaCatalogMember[]; readonly names: readonly string[]; } +function kindOf(raw: unknown): string { + if (raw === null || raw === undefined) return "nil"; + if (Array.isArray(raw)) return "array"; + if (raw instanceof WireFloat) return "float"; + if (typeof raw === "bigint") return "integer"; + return typeof raw; +} + +function refuse(what: string, found: string): never { + throw new Error(`expected ${what}, found ${found}`); +} + function expect(raw: unknown, what: string, ok: boolean): asserts ok { - if (!ok) { - throw new Error(`expected ${what}, found ${raw === null ? "nil" : Array.isArray(raw) ? "array" : typeof raw}`); - } + if (!ok) refuse(what, kindOf(raw)); } export const string: Codec = { + typeName: "System.String", encode: (value) => (expect(value, "a string", typeof value === "string"), value), decode: (raw) => (expect(raw, "a string", typeof raw === "string"), raw as string), }; -export const number: Codec = { - encode: (value) => (expect(value, "a number", Number.isFinite(value)), value), - decode: (raw) => (expect(raw, "a number", Number.isFinite(raw)), raw as number), -}; - export const boolean: Codec = { + typeName: "System.Boolean", + valueType: true, encode: (value) => (expect(value, "a boolean", typeof value === "boolean"), value), decode: (raw) => (expect(raw, "a boolean", typeof raw === "boolean"), raw as boolean), }; export const bytes: Codec = { + typeName: "System.Byte[]", encode: (value) => (expect(value, "bytes", value instanceof Uint8Array), value), decode: (raw) => (expect(raw, "bytes", raw instanceof Uint8Array), raw as Uint8Array), }; -export function list(item: Codec): Codec { +// A C# integer slot: written as the smallest MessagePack integer and read from any integer encoding in range. A float is +// refused even when it holds a whole number, as the reference refuses it. +function integerCodec(typeName: string, what: string, low: bigint, high: bigint): Codec { + const inRange = (value: number | bigint) => value >= low && value <= high; return { + typeName, + valueType: true, + encode(value) { + if (typeof value === "bigint") { + if (!inRange(value)) refuse(what, "an integer out of range"); + return value; + } + if (typeof value !== "number") refuse(what, kindOf(value)); + if (!Number.isSafeInteger(value)) { + refuse(what, Number.isInteger(value) ? "a number beyond 2^53, which needs a bigint" : "a non-integer number"); + } + if (!inRange(value)) refuse(what, "an integer out of range"); + return value; + }, + decode(raw) { + if (typeof raw !== "number" && typeof raw !== "bigint") refuse(what, kindOf(raw)); + if (!inRange(raw)) refuse(what, "an integer out of range"); + return raw; + }, + }; +} + +// C# int. Always a number. +export const int32 = integerCodec("System.Int32", "an int32", -(2n ** 31n), 2n ** 31n - 1n) as Codec; +// C# long. A number when the value is a safe integer, a bigint beyond 2^53 (so 64-bit values never lose precision). +export const int64 = integerCodec("System.Int64", "an int64", -(2n ** 63n), 2n ** 63n - 1n); + +function floatCodec(typeName: string, wide: boolean): Codec { + const what = wide ? "a float64" : "a float32"; + return { + typeName, + valueType: true, + encode(value) { + expect(value, what, typeof value === "number"); + return new WireFloat(value, wide); + }, + decode(raw) { + if (raw instanceof WireFloat) return wide ? raw.value : Math.fround(raw.value); + if (typeof raw === "number" || typeof raw === "bigint") return wide ? Number(raw) : Math.fround(Number(raw)); + return refuse(what, kindOf(raw)); + }, + }; +} + +// C# float, written as MessagePack float 32. +export const float32 = floatCodec("System.Single", false); +// C# double, written as MessagePack float 64. +export const float64 = floatCodec("System.Double", true); + +function sequence(item: Codec, nameOf: (itemName: string) => string): Codec { + return { + get typeName() { return item.typeName === undefined ? undefined : nameOf(item.typeName); }, encode(value) { expect(value, "an array", Array.isArray(value)); return value.map((element, index) => at(`element ${index}`, () => item.encode(element))); @@ -54,8 +123,22 @@ export function list(item: Codec): Codec { }; } +// C# List. +export function list(item: Codec): Codec { + return sequence(item, (name) => `System.Collections.Generic.List<${name}>`); +} + +// C# T[]. The same wire as list. +export function array(item: Codec): Codec { + return sequence(item, (name) => `${name}[]`); +} + +// A slot that may be nil. For a value type the reference's name is Nullable; for a reference type it is unchanged. export function nullable(inner: Codec): Codec { return { + get typeName() { + return inner.typeName !== undefined && inner.valueType ? `System.Nullable<${inner.typeName}>` : inner.typeName; + }, encode: (value) => (value === null || value === undefined ? null : inner.encode(value)), decode: (raw) => (raw === null ? null : inner.decode(raw)), }; @@ -65,6 +148,8 @@ export function lazy(get: () => Codec): Codec { let resolved: Codec | undefined; const codec = () => (resolved ??= get()); return { + get typeName() { return codec().typeName; }, + get valueType() { return codec().valueType; }, encode: (value) => codec().encode(value), decode: (raw) => codec().decode(raw), }; @@ -85,13 +170,19 @@ type SlotSpec = type SlotsSpec = Readonly }[keyof TValue & string]>>; -function slotKeys(spec: Readonly>, owner: string): number[] { +const INT32_MAX = 2 ** 31 - 1; + +// The declared keys, ascending. An arm key is an Int32 in the reference, so a union refuses a wider key at definition. +function slotKeys(spec: Readonly>, owner: string, max = Number.MAX_SAFE_INTEGER): number[] { // Object.keys lists integer keys in ascending order, so the declared keys come back sorted. return Object.keys(spec).map((text) => { const key = Number(text); if (!Number.isInteger(key) || key < 0 || String(key) !== text) { throw new Error(`${owner} declares the key "${text}", which is not a non-negative integer.`); } + if (key > max) { + throw new Error(`${owner} declares the key ${key}, which is outside Int32.`); + } return key; }); } @@ -107,7 +198,7 @@ function pascalCase(name: string): string { // A positional record: slot `key` holds field `name`. Gaps are written nil and skipped on read, extra trailing slots // are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that // declares a default also takes the default (cultcache-py's rule). -export function slots(spec: SlotsSpec): SlotsCodec { +export function slots(spec: SlotsSpec, typeName?: string): SlotsCodec { const keys = slotKeys(spec, "A slot record"); if (keys.length === 0) { throw new Error("A slot record declares no slots."); @@ -129,7 +220,15 @@ export function slots(spec: SlotsSpec): SlotsCode return { names: fields.map((field) => field.name), - members: fields.map((field) => ({ slot: field.key, memberName: pascalCase(field.name), typeName: "System.Object" })), + typeName, + get members() { + return fields.map((field) => { + if (field.codec.typeName === undefined) { + throw new Error(`${field.name} (slot ${field.key}) has no C# type name for the schema catalog; give its union or record a typeName, or give the document its members.`); + } + return { slot: field.key, memberName: pascalCase(field.name), typeName: field.codec.typeName }; + }); + }, encode(value) { expect(value, "a record", typeof value === "object" && value !== null); const out: unknown[] = new Array(length).fill(null); @@ -170,8 +269,9 @@ type ArmSpec = readonly [armName: TKind, body: SlotsCodec< export function union( name: string, arms: Readonly>>, + typeName?: string, ): Codec { - const keys = slotKeys(arms, `Union ${name}`); + const keys = slotKeys(arms, `Union ${name}`, INT32_MAX); if (keys.length === 0) { throw new Error(`Union ${name} declares no arms.`); } @@ -191,6 +291,7 @@ export function union( const declared = keys.join(", "); return { + typeName, encode(value) { if (value === null || value === undefined) { return null; @@ -210,10 +311,13 @@ export function union( throw new Error(`Union ${name}: expected [key, slots] and found something else.`); } const key: unknown = raw[0]; - if (typeof key !== "number" || !Number.isInteger(key)) { + if (typeof key !== "number" && typeof key !== "bigint") { throw new Error(`Union ${name}: the arm key is not an integer (declared keys: ${declared}).`); } - const arm = byKey.get(key); + if (key < -(2 ** 31) || key > INT32_MAX) { + throw new Error(`Union ${name}: the arm key is outside Int32 (declared keys: ${declared}).`); + } + const arm = byKey.get(Number(key)); if (arm === undefined) { throw new Error(`Union ${name}: unknown arm key ${key} (declared keys: ${declared}).`); } @@ -226,7 +330,7 @@ export function union( // The formatter a document gets from a slot codec: the codec's value as one MessagePack payload. export function codecFormatter(codec: SlotsCodec): CultCacheDocumentFormatter { return { - encode: (value) => encode(codec.encode(value)), - decode: (payload) => codec.decode(decode(payload)), + encode: (value) => writeWire(codec.encode(value)), + decode: (payload) => codec.decode(readWire(payload)), }; } diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index a1c0a3d4..94247d49 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,6 +1,4 @@ -import { decode, encode } from "@msgpack/msgpack"; - -import { codecFormatter } from "./codec"; +import { resolveDocumentFormatter, resolveDocumentMembers } from "./document"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, @@ -677,10 +675,7 @@ export class CultCache { } #createFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { - const userFormatter = definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : { - encode: (value: unknown) => encode(value), - decode: (payload: Uint8Array) => decode(payload), - }); + const userFormatter = resolveDocumentFormatter(definition); return { encode: (value: unknown) => { @@ -700,7 +695,7 @@ export class CultCache { const canonicalSchemaJson = definition.canonicalSchemaJson ?? JSON.stringify({ schemaName, schemaVersion, - members: [...(definition.members ?? [])] + members: [...resolveDocumentMembers(definition)] .sort((left, right) => left.slot - right.slot) .map((member) => ({ slot: member.slot, @@ -725,7 +720,7 @@ export class CultCache { contentHash: definition.contentHash ?? schemaId, canonicalSchemaJson, compatibleSchemaIds, - members: [...(definition.members ?? [])] + members: [...resolveDocumentMembers(definition)] .sort((left, right) => left.slot - right.slot) .map((member) => ({ slot: member.slot, diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 70cdd7b2..276e7caa 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -1,5 +1,10 @@ +import { decode, encode } from "@msgpack/msgpack"; + +import { codecFormatter } from "./codec"; import type { AnyCultCacheDocumentDefinition, + CultCacheDocumentFormatter, + CultCacheSchemaCatalogMember, CultCacheDocumentDefinition, CultCacheDocumentRegistry, CultCacheSchema, @@ -66,24 +71,38 @@ export function defineDocumentType( } } - if (definition.codec) { - const codec = definition.codec; - if (definition.formatter) { - throw new Error(`CultCache document type "${definition.type}" declares both a codec and a formatter.`); - } - const indexAlias = (slotName: string) => - !Array.isArray(definition.indexes) && definition.indexes - ? Object.entries(definition.indexes).find(([, accessor]) => accessor === slotName)?.[0] ?? null - : null; - const members = definition.members ?? codec.members.map((member, index) => ({ - ...member, - isName: definition.name === codec.names[index], - indexAlias: indexAlias(codec.names[index]), - })); - return Object.freeze({ ...definition, members }); - } + resolveDocumentFormatter(definition); + return Object.freeze(definition.codec ? { ...definition, members: resolveDocumentMembers(definition) } : { ...definition }); +} - return Object.freeze({ ...definition }); +// The one place that decides how a document is encoded: its own formatter, or its codec's, or a plain map. A codec and a +// formatter are two answers to one question, so a definition that carries both is refused wherever it enters (the +// builder resolves through here as well). +export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { + if (definition.codec && definition.formatter) { + throw new Error(`CultCache document type "${definition.type}" declares both a codec and a formatter.`); + } + return definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : { + encode: (value: unknown) => encode(value), + decode: (payload: Uint8Array) => decode(payload), + }); +} + +// The schema members of a document: the ones it declares, else those its codec derives, else none. +export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinition): readonly CultCacheSchemaCatalogMember[] { + if (definition.members || !definition.codec) { + return definition.members ?? []; + } + const codec = definition.codec; + const indexAlias = (slotName: string) => + !Array.isArray(definition.indexes) && definition.indexes + ? Object.entries(definition.indexes).find(([, accessor]) => accessor === slotName)?.[0] ?? null + : null; + return codec.members.map((member, index) => ({ + ...member, + isName: definition.name === codec.names[index], + indexAlias: indexAlias(codec.names[index]), + })); } export function defineDocumentRegistry< diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts new file mode 100644 index 00000000..d57ff049 --- /dev/null +++ b/packages/cultcache-ts/src/wire.ts @@ -0,0 +1,130 @@ +// The slice of MessagePack a codec value uses: nil, booleans, integers, floats, strings, binaries and arrays. A codec +// must tell an integer from a float and keep 64-bit integers whole, which a general decoder that hands back JS numbers +// cannot; so the codec path owns its reader and writer. An integer decodes to a number, or to a bigint when it is +// outside the safe range. A float decodes to a WireFloat, whatever its value. + +export class WireFloat { + constructor(readonly value: number, readonly wide: boolean) {} +} + +const SAFE_MIN = BigInt(Number.MIN_SAFE_INTEGER); +const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER); +const utf8 = new TextEncoder(); +const utf8Decoder = new TextDecoder(); + +export function writeWire(tree: unknown): Uint8Array { + const parts: number[] = []; + put(tree, parts); + return Uint8Array.from(parts); +} + +function be(parts: number[], bytes: number, value: bigint): void { + for (let shift = BigInt((bytes - 1) * 8); shift >= 0n; shift -= 8n) { + parts.push(Number((value >> shift) & 0xffn)); + } +} + +function putInt(value: bigint, parts: number[]): void { + if (value >= 0n) { + if (value < 128n) parts.push(Number(value)); + else if (value < 0x100n) { parts.push(0xcc); be(parts, 1, value); } + else if (value < 0x10000n) { parts.push(0xcd); be(parts, 2, value); } + else if (value < 0x100000000n) { parts.push(0xce); be(parts, 4, value); } + else if (value < 0x10000000000000000n) { parts.push(0xcf); be(parts, 8, value); } + else throw new Error("cannot write an integer wider than 64 bits."); + } else if (value >= -32n) parts.push(0x100 + Number(value)); + else if (value >= -0x80n) { parts.push(0xd0); be(parts, 1, value & 0xffn); } + else if (value >= -0x8000n) { parts.push(0xd1); be(parts, 2, value & 0xffffn); } + else if (value >= -0x80000000n) { parts.push(0xd2); be(parts, 4, value & 0xffffffffn); } + else if (value >= -0x8000000000000000n) { parts.push(0xd3); be(parts, 8, value & 0xffffffffffffffffn); } + else throw new Error("cannot write an integer wider than 64 bits."); +} + +function putLength(length: number, parts: number[], fix: number, fixMax: number, codes: readonly [number, number, number]): void { + if (length <= fixMax) parts.push(fix | length); + else if (codes[0] !== 0 && length < 0x100) { parts.push(codes[0]); be(parts, 1, BigInt(length)); } + else if (length < 0x10000) { parts.push(codes[1]); be(parts, 2, BigInt(length)); } + else { parts.push(codes[2]); be(parts, 4, BigInt(length)); } +} + +function put(node: unknown, parts: number[]): void { + if (node === null || node === undefined) parts.push(0xc0); + else if (typeof node === "boolean") parts.push(node ? 0xc3 : 0xc2); + else if (typeof node === "bigint") putInt(node, parts); + else if (typeof node === "number") { + if (!Number.isSafeInteger(node)) throw new Error("cannot write a number that is not a safe integer; floats are WireFloat."); + putInt(BigInt(node), parts); + } else if (node instanceof WireFloat) { + const view = new DataView(new ArrayBuffer(8)); + if (node.wide) { view.setFloat64(0, node.value); parts.push(0xcb); } else { view.setFloat32(0, node.value); parts.push(0xca); } + for (let index = 0; index < (node.wide ? 8 : 4); index += 1) parts.push(view.getUint8(index)); + } else if (typeof node === "string") { + const text = utf8.encode(node); + putLength(text.length, parts, 0xa0, 31, [0xd9, 0xda, 0xdb]); + for (const byte of text) parts.push(byte); + } else if (node instanceof Uint8Array) { + putLength(node.length, parts, 0, -1, [0xc4, 0xc5, 0xc6]); + for (const byte of node) parts.push(byte); + } else if (Array.isArray(node)) { + putLength(node.length, parts, 0x90, 15, [0, 0xdc, 0xdd]); + for (const element of node) put(element, parts); + } else { + throw new Error("cannot write this kind of value."); + } +} + +export function readWire(payload: Uint8Array): unknown { + const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength); + let offset = 0; + + const take = (count: number): number => { + if (count < 0 || offset + count > payload.length) throw new Error("the MessagePack payload is truncated."); + offset += count; + return offset - count; + }; + const integer = (value: bigint): number | bigint => (value >= SAFE_MIN && value <= SAFE_MAX ? Number(value) : value); + const string = (length: number): string => utf8Decoder.decode(payload.subarray(take(length), offset)); + const binary = (length: number): Uint8Array => payload.slice(take(length), offset); + const array = (length: number): unknown[] => { + if (length > payload.length - offset) throw new Error("the MessagePack payload is truncated."); + const out: unknown[] = []; + for (let index = 0; index < length; index += 1) out.push(next()); + return out; + }; + + function next(): unknown { + const head = payload[take(1)]; + if (head < 0x80) return head; + if (head >= 0xe0) return head - 0x100; + if (head >= 0xa0 && head <= 0xbf) return string(head & 0x1f); + if (head >= 0x90 && head <= 0x9f) return array(head & 0x0f); + switch (head) { + case 0xc0: return null; + case 0xc2: return false; + case 0xc3: return true; + case 0xc4: return binary(view.getUint8(take(1))); + case 0xc5: return binary(view.getUint16(take(2))); + case 0xc6: return binary(view.getUint32(take(4))); + case 0xca: return new WireFloat(view.getFloat32(take(4)), false); + case 0xcb: return new WireFloat(view.getFloat64(take(8)), true); + case 0xcc: return view.getUint8(take(1)); + case 0xcd: return view.getUint16(take(2)); + case 0xce: return view.getUint32(take(4)); + case 0xcf: return integer(view.getBigUint64(take(8))); + case 0xd0: return view.getInt8(take(1)); + case 0xd1: return view.getInt16(take(2)); + case 0xd2: return view.getInt32(take(4)); + case 0xd3: return integer(view.getBigInt64(take(8))); + case 0xd9: return string(view.getUint8(take(1))); + case 0xda: return string(view.getUint16(take(2))); + case 0xdb: return string(view.getUint32(take(4))); + case 0xdc: return array(view.getUint16(take(2))); + case 0xdd: return array(view.getUint32(take(4))); + default: throw new Error("the MessagePack payload holds a type a codec does not read (map, extension or reserved)."); + } + } + + const tree = next(); + if (offset !== payload.length) throw new Error("the MessagePack payload has bytes after its value."); + return tree; +} diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 8aca877d..52dfd76a 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -11,11 +11,15 @@ import { z } from "zod"; import { codecs } from "../src"; import type { Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; +import { readWire, WireFloat, writeWire } from "../src/wire"; import { defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { CacheBackingStore, CultCacheEnvelope } from "../src/types"; -const { boolean, bytes, lazy, list, nullable, number, slots, string, union, unit } = codecs; +const { array, boolean, bytes, float32, float64, int32, int64, lazy, list, nullable, slots, string, union, unit } = codecs; + +// The names the C# reference gives the fixture's types (CultSchemaTypeNames.FromType), as union-surface.cc carries them. +const CS = "GameCult.Caching.Tests.UnionParityVectorTests+"; // The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected // outcome is read from those committed files; only the value the payload must decode to is spelled here. @@ -50,7 +54,7 @@ interface Surface { missing: Node | null; } -const composition = slots({ 0: ["name", string], 1: ["body", lazy(() => node)] }); +const composition = slots({ 0: ["name", string], 1: ["body", lazy(() => node)] }, `${CS}Composition`); const node: Codec = union("Node", { 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })], 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })], @@ -58,29 +62,29 @@ const node: Codec = union("Node", { 0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))], })], 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], -}); +}, `${CS}Node`); const outcome = union("Outcome", { 0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })], - 1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", number] })], + 1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", int32] })], 2: ["Denied", unit], -}); -const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", number] }); -const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", number] }); +}, `${CS}Outcome`); +const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] }); +const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] }); const item = union("Item", { 0: ["Gem", slots<{ name: string }>({ 0: ["name", string] })], 1: ["Tool", toolArm], 2: ["Part", partArm], -}); -const craftedItem = union("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }); +}, `${CS}Item`); +const craftedItem = union("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }, `${CS}CraftedItem`); const surfaceCodec = slots({ 0: ["id", string], - 1: ["version", number], + 1: ["version", int32], 2: ["root", node], 3: ["outcomes", list(outcome)], 4: ["part", item], 5: ["crafted", craftedItem], 6: ["missing", node, null], -}); +}, `${CS}UnionSurface`); const nodeSchema: z.ZodType = z.lazy(() => z.union([ @@ -175,10 +179,10 @@ async function outcomeEntries() { test("the union surface vector decodes to the constructed value and re-encodes byte-identical", async () => { const committed = await read("union-surface.msgpack"); - const decoded = surfaceCodec.decode(decode(committed)); + const decoded = surfaceCodec.decode(readWire(committed)); assert.deepEqual(decoded, golden); - assert.equal(hex(encode(surfaceCodec.encode(decoded))), hex(committed)); - assert.equal(hex(encode(surfaceCodec.encode(golden))), hex(committed)); + assert.equal(hex(writeWire(surfaceCodec.encode(decoded))), hex(committed)); + assert.equal(hex(writeWire(surfaceCodec.encode(golden))), hex(committed)); }); test("a unit arm is written [key, []] and its payload must be an array", () => { @@ -219,7 +223,7 @@ test("every outcome vector is listed and holds: refusals name the union and key, refused += 1; if (Buffer.from(payload).includes(CANARY)) carrying.push(file); assert.throws( - () => surfaceCodec.decode(decode(payload)), + () => surfaceCodec.decode(readWire(payload)), (error: Error) => { assert.ok(!error.message.includes(CANARY), `${file}: the refusal echoed the payload`); if (detail === "arity") { @@ -234,7 +238,7 @@ test("every outcome vector is listed and holds: refusals name the union and key, } else { readOnly += 1; assert.equal(expected, "read-only"); - assert.deepEqual(surfaceCodec.decode(decode(payload)), golden, `${file}: the extra slot is read past`); + assert.deepEqual(surfaceCodec.decode(readWire(payload)), golden, `${file}: the extra slot is read past`); } } assert.deepEqual([refused, readOnly], [5, 4]); @@ -243,7 +247,7 @@ test("every outcome vector is listed and holds: refusals name the union and key, test("a refusal lists the declared keys, so a key only the outer union declares is unknown to the inner", async () => { const entry = (await outcomeEntries()).find((candidate) => candidate.file === "inner-union-outer-key.msgpack")!; - const payload = decode(await read(entry.file)) as unknown[]; + const payload = readWire(await read(entry.file)) as unknown[]; assert.throws( () => surfaceCodec.decode(payload), /Union CraftedItem: unknown arm key 2 \(declared keys: 0, 1\)\./u, @@ -275,7 +279,7 @@ test("a union encodes by kind and refuses a kind it does not declare without ech }); test("slots: a gap writes nil, a missing slot reads its default, one without a default is refused naming the slot", () => { - const record = slots<{ a: string; c: string; d: number }>({ 0: ["a", string], 2: ["c", string], 4: ["d", number, 9] }); + const record = slots<{ a: string; c: string; d: number }>({ 0: ["a", string], 2: ["c", string], 4: ["d", int32, 9] }); assert.deepEqual(record.encode({ a: "x", c: "y", d: 1 }), ["x", null, "y", null, 1]); assert.deepEqual(record.decode(["x", null, "y"]), { a: "x", c: "y", d: 9 }); assert.deepEqual(record.decode(["x", 5, "y", null, null]), { a: "x", c: "y", d: 9 }); @@ -301,9 +305,15 @@ test("a slot refusal names the field and slot and never the value", () => { test("leaf codecs refuse the wrong shape on both sides, saying what was expected and what kind was found", () => { const table: [string, Codec, string, unknown, string][] = [ ["string", string, "a string", 5, "number"], - ["number", number, "a number", "5", "string"], - ["number", number, "a number", Number.POSITIVE_INFINITY, "number"], - ["number", number, "a number", Number.NaN, "number"], + ["int32", int32, "an int32", "5", "string"], + ["int32", int32, "an int32", Number.POSITIVE_INFINITY, "a non-integer number"], + ["int32", int32, "an int32", Number.NaN, "a non-integer number"], + ["int32", int32, "an int32", 1.5, "a non-integer number"], + ["int32", int32, "an int32", 2 ** 31, "an integer out of range"], + ["int64", int64, "an int64", 2 ** 53, "a number beyond 2^53, which needs a bigint"], + ["int64", int64, "an int64", 2n ** 63n, "an integer out of range"], + ["float64", float64, "a float64", "1.5", "string"], + ["float32", float32, "a float32", null, "nil"], ["boolean", boolean, "a boolean", "true", "string"], ["bytes", bytes, "bytes", [1, 2], "array"], ["string", string, "a string", null, "nil"], @@ -311,25 +321,27 @@ test("leaf codecs refuse the wrong shape on both sides, saying what was expected ]; for (const [label, codec, what, bad, found] of table) { for (const side of ["encode", "decode"] as const) { + // Decode sees the wire's shapes: a JS float or a number beyond 2^53 never arrives from a payload. + if (side === "decode" && (typeof bad === "number" && (!Number.isInteger(bad) || Math.abs(bad) >= 2 ** 53))) continue; assert.throws(() => codec[side](bad), new RegExp(`^Error: expected ${what}, found ${found}$`, "u"), `${label} ${side}`); } } - for (const [codec, good] of [[string, "s"], [number, 1.5], [boolean, false], [bytes, new Uint8Array([7])], [list(number), [1, 2]]] as [Codec, unknown][]) { + for (const [codec, good] of [[string, "s"], [int32, 1], [int64, 5], [float64, 1.5], [float32, 1.5], [boolean, false], [bytes, new Uint8Array([7])], [list(int32), [1, 2]]] as [Codec, unknown][]) { assert.deepEqual(codec.decode(codec.encode(good)), good); } }); test("a list element refusal names the element, and nullable writes nil for null and undefined", () => { - const numbers = list(number); - assert.throws(() => numbers.encode([1, "x" as unknown as number]), /^Error: element 1: expected a number, found string$/u); - assert.throws(() => numbers.decode([1, 2, null]), /^Error: element 2: expected a number, found nil$/u); - const maybe = nullable(number); + const numbers = list(int32); + assert.throws(() => numbers.encode([1, "x" as unknown as number]), /^Error: element 1: expected an int32, found string$/u); + assert.throws(() => numbers.decode([1, 2, null]), /^Error: element 2: expected an int32, found nil$/u); + const maybe = nullable(int32); assert.equal(maybe.encode(null), null); assert.equal(maybe.encode(undefined as unknown as null), null); assert.equal(maybe.encode(0), 0); assert.equal(maybe.decode(null), null); assert.equal(maybe.decode(0), 0); - assert.throws(() => maybe.decode("x"), /expected a number/u); + assert.throws(() => maybe.decode("x"), /expected an int32/u); }); test("a record refuses what is not a record and names the field and slot of a bad value", () => { @@ -384,8 +396,8 @@ test("a codec supplies the document's catalog members unless it is given its own indexes: { byB: "the_b" }, }); assert.deepEqual(document.members, [ - { slot: 0, memberName: "A", typeName: "System.Object", isName: true, indexAlias: null }, - { slot: 3, memberName: "TheB", typeName: "System.Object", isName: false, indexAlias: "byB" }, + { slot: 0, memberName: "A", typeName: "System.String", isName: true, indexAlias: null }, + { slot: 3, memberName: "TheB", typeName: "System.String", isName: false, indexAlias: "byB" }, ]); const given = [{ slot: 9, memberName: "Mine", typeName: "System.String" }]; const own = defineDocumentType({ type: "t", schema: z.object({ a: z.string() }), codec: slots<{ a: string }>({ 0: ["a", string] }), members: given }); From b4f41385e379a217d8440897f585bc5fe57d4604 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:51:51 +0200 Subject: [PATCH 10/44] cultcache-ts tests: arm keys, wire, catalog parity, one codec-or-formatter owner (not yet run; does not build until the numeric table lands) Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/document.ts | 2 +- .../cultcache-ts/test/union-parity.test.ts | 107 +++++++++++++++++- 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 276e7caa..a09fec81 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -100,7 +100,7 @@ export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinitio : null; return codec.members.map((member, index) => ({ ...member, - isName: definition.name === codec.names[index], + isName: (definition.name as unknown) === codec.names[index], indexAlias: indexAlias(codec.names[index]), })); } diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 52dfd76a..ab75c000 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -207,6 +207,14 @@ test("union-surface.cc reads through SingleFileMessagePackBackingStore and write await fresh.put(document, envelope.key, golden); const [rewritten] = await new SingleFileMessagePackBackingStore(written).pullAll(); assert.equal(hex(rewritten.payload), hex(envelope.payload)); + + // The reference refuses a store whose schema catalog names another type for a slot, so the catalog TS writes must be + // the one C# wrote: same canonical schema text, same member names, slots and types. + assert.equal(rewritten.catalogEntry!.canonicalSchemaJson, envelope.catalogEntry!.canonicalSchemaJson); + const shape = (members: readonly { slot: number; memberName: string; typeName: string; isName?: boolean }[]) => + members.map((member) => [member.slot, member.memberName, member.typeName, member.isName === true]); + assert.deepEqual(shape(rewritten.catalogEntry!.members!), shape(envelope.catalogEntry!.members!)); + assert.equal(rewritten.catalogEntry!.members!.length, 7); }); test("every outcome vector is listed and holds: refusals name the union and key, never the payload", async () => { @@ -323,7 +331,7 @@ test("leaf codecs refuse the wrong shape on both sides, saying what was expected for (const side of ["encode", "decode"] as const) { // Decode sees the wire's shapes: a JS float or a number beyond 2^53 never arrives from a payload. if (side === "decode" && (typeof bad === "number" && (!Number.isInteger(bad) || Math.abs(bad) >= 2 ** 53))) continue; - assert.throws(() => codec[side](bad), new RegExp(`^Error: expected ${what}, found ${found}$`, "u"), `${label} ${side}`); + assert.throws(() => codec[side](bad), { message: `expected ${what}, found ${found}` }, `${label} ${side}`); } } for (const [codec, good] of [[string, "s"], [int32, 1], [int64, 5], [float64, 1.5], [float32, 1.5], [boolean, false], [bytes, new Uint8Array([7])], [list(int32), [1, 2]]] as [Codec, unknown][]) { @@ -426,6 +434,103 @@ test("bytes and nullable round-trip through a slot record", () => { assert.throws(() => record.decode([new Uint8Array(), 4]), /maybe \(slot 1\): expected a string/u); }); +const fromHex = (text: string) => new Uint8Array(Buffer.from(text, "hex")); + +// Arm keys are integers within Int32 in the reference: a float key is refused even when it holds a whole number, and a +// key outside Int32 is refused whether it is declared or read. Every payload below is [key, [7]] or [key, ["wait"]]. +test("an arm key is an integer in any MessagePack integer encoding, and a float-typed key is refused", () => { + const completed = { kind: "Completed", sourceVersion: 7 }; + for (const key of ["01", "cc01", "cd0001", "ce00000001", "cf0000000000000001", "d001", "d10001", "d200000001", "d30000000000000001"]) { + assert.deepEqual(outcome.decode(readWire(fromHex(`92${key}9107`))), completed, key); + } + for (const key of ["ca3f800000", "cb3ff0000000000000"]) { + assert.throws(() => outcome.decode(readWire(fromHex(`92${key}9107`))), { message: "Union Outcome: the arm key is not an integer (declared keys: 0, 1, 2)." }, key); + } + assert.throws(() => outcome.decode(readWire(fromHex("92cb800000000000000091a477616974"))), /^Error: Union Outcome: the arm key is not an integer/u, "-0.0 is a float, not key 0"); + for (const key of ["ce80000000", "cf8000000000000000", "d30000000080000000"]) { + assert.throws(() => outcome.decode(readWire(fromHex(`92${key}9107`))), { message: "Union Outcome: the arm key is outside Int32 (declared keys: 0, 1, 2)." }, key); + } + assert.throws(() => outcome.decode(readWire(fromHex("92d2800000009107"))), { message: "Union Outcome: unknown arm key -2147483648 (declared keys: 0, 1, 2)." }); + assert.throws(() => outcome.decode(readWire(fromHex("92ff9107"))), { message: "Union Outcome: unknown arm key -1 (declared keys: 0, 1, 2)." }); +}); + +test("a union refuses a declared arm key outside Int32", () => { + assert.doesNotThrow(() => union<{ kind: "A" }>("U", { 2147483647: ["A", unit] })); + assert.throws(() => union<{ kind: "A" }>("U", { 2147483648: ["A", unit] }), { message: "Union U declares the key 2147483648, which is outside Int32." }); + assert.throws(() => union<{ kind: "A" }>("U", { 3000000000: ["A", unit] }), /outside Int32/u); +}); + +test("the wire reader keeps 64-bit integers whole, tells a float from an integer, and refuses what is not a value", () => { + assert.equal(readWire(fromHex("cf0020000000000001")), 2n ** 53n + 1n); + assert.equal(readWire(fromHex("cf001fffffffffffff")), 2 ** 53 - 1); + assert.equal(readWire(fromHex("d38000000000000000")), -(2n ** 63n)); + assert.equal(readWire(fromHex("cfffffffffffffffff")), 2n ** 64n - 1n); + assert.ok(readWire(fromHex("cb3ff0000000000000")) instanceof WireFloat); + assert.ok(readWire(fromHex("ca3f800000")) instanceof WireFloat); + assert.equal(readWire(fromHex("01")), 1); + assert.throws(() => readWire(fromHex("92a161")), /truncated/u, "an array that promises more than the payload holds"); + assert.throws(() => readWire(fromHex("dd")), /truncated/u); + assert.throws(() => readWire(fromHex("ddffffffff")), /truncated/u, "a huge array header reserves nothing"); + assert.throws(() => readWire(fromHex("a5616263")), /truncated/u); + assert.throws(() => readWire(fromHex("0102")), /bytes after its value/u); + assert.throws(() => readWire(fromHex("81a161a162")), /map, extension or reserved/u); + assert.throws(() => readWire(fromHex("c1")), /map, extension or reserved/u); + assert.throws(() => readWire(fromHex("d4010a")), /map, extension or reserved/u); +}); + +test("the wire writer writes the smallest integer, keeps a float's width, and round-trips what it reads", () => { + const tree = [0, 127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296, -1, -32, -33, -128, -129, -32768, -32769, -2147483648, -2147483649, + 2n ** 53n, 2n ** 64n - 1n, -(2n ** 63n), new WireFloat(1.5, false), new WireFloat(1.5, true), "", "a".repeat(31), "a".repeat(32), "a".repeat(256), + new Uint8Array(0), new Uint8Array(300), [], null, true, false, "héllo"]; + assert.deepEqual(readWire(writeWire(tree)), tree); + assert.equal(hex(writeWire(5)), "05"); + assert.equal(hex(writeWire(128)), "cc80"); + assert.equal(hex(writeWire(-33)), "d0df"); + assert.equal(hex(writeWire(2n ** 53n + 2n)), "cf0020000000000002"); + assert.equal(hex(writeWire(new WireFloat(3, true))), "cb4008000000000000"); + assert.equal(hex(writeWire(new WireFloat(1.5, false))), "ca3fc00000"); + assert.equal(hex(writeWire(new Array(16).fill(null))), `dc0010${"c0".repeat(16)}`); + assert.throws(() => writeWire(1.5), /not a safe integer/u); + assert.throws(() => writeWire(2 ** 53), /not a safe integer/u); + assert.throws(() => writeWire(2n ** 64n), /wider than 64 bits/u); + assert.throws(() => writeWire({}), /this kind of value/u); +}); + +test("a codec and a formatter are refused together wherever the definition enters", () => { + const both = { type: "t", schema: surfaceSchema, codec: surfaceCodec, formatter: { encode: () => new Uint8Array(), decode: () => golden } }; + assert.throws(() => defineDocumentType(both), /declares both a codec and a formatter/u); + assert.throws(() => CultCache.builder().withDocumentType(both), /declares both a codec and a formatter/u); + assert.throws(() => CultCache.builder().withRegistry([both]), /declares both a codec and a formatter/u); +}); + +test("a definition that skips defineDocumentType still writes its codec's bytes and its derived catalog", async () => { + const raw = { type: "tests.union_surface", schemaId: "tests.union_surface.v1", schema: surfaceSchema, codec: surfaceCodec, name: "id" as const }; + const holder = memoryStore([]); + const cache = CultCache.builder().withDocumentType(raw).withGenericStore(holder).build(); + await cache.put(raw, "surface", golden); + assert.equal(hex(holder.entries[0].payload), hex(await read("union-surface.msgpack"))); + const [snapshot] = cache.snapshot(); + assert.deepEqual(snapshot.catalogEntry!.members!.map((member) => member.typeName).slice(0, 2), ["System.String", "System.Int32"]); +}); + +test("every leaf and wrapper carries the reference's type name, and a catalog is not derived from a codec that cannot name a slot", () => { + assert.deepEqual( + [string, boolean, bytes, int32, int64, float32, float64].map((codec) => codec.typeName), + ["System.String", "System.Boolean", "System.Byte[]", "System.Int32", "System.Int64", "System.Single", "System.Double"], + ); + assert.equal(list(int64).typeName, "System.Collections.Generic.List"); + assert.equal(array(string).typeName, "System.String[]"); + assert.equal(nullable(int32).typeName, "System.Nullable"); + assert.equal(nullable(string).typeName, "System.String"); + assert.equal(nullable(list(string)).typeName, "System.Collections.Generic.List"); + assert.equal(list(union<{ kind: "A" }>("U", { 0: ["A", unit] })).typeName, undefined); + const unnamed = slots<{ id: string; what: { kind: "A" } | null }>({ 0: ["id", string], 1: ["what", union<{ kind: "A" }>("U", { 0: ["A", unit] })] }); + assert.throws(() => unnamed.members, /^Error: what \(slot 1\) has no C# type name for the schema catalog/u); + assert.throws(() => defineDocumentType({ type: "t", schema: z.any(), codec: unnamed as never }), /what \(slot 1\) has no C# type name/u); + const given = defineDocumentType({ type: "t", schema: z.any(), codec: unnamed as never, members: [{ slot: 0, memberName: "Id", typeName: "System.String" }] }); + assert.equal(given.members!.length, 1); +}); + function envelopeOf(payload: Uint8Array): CultCacheEnvelope { return { key: "surface", type: "tests.union_surface", payload, storedAt: new Date().toISOString() }; } From 17df29cc9c50cf96b6b0d24b46ae07827c372ae3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:55:22 +0200 Subject: [PATCH 11/44] cultcache-ts tests: numeric leaves held to the C# reference's reads and writes The table is the reference's own answer for 120 one-slot reads and 46 writes, from a job-only probe of src/. Co-Authored-By: Claude Opus 5.5 --- .../test/reference-numeric-table.ts | 176 ++++++++++++++++++ .../cultcache-ts/test/union-parity.test.ts | 48 ++++- 2 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 packages/cultcache-ts/test/reference-numeric-table.ts diff --git a/packages/cultcache-ts/test/reference-numeric-table.ts b/packages/cultcache-ts/test/reference-numeric-table.ts new file mode 100644 index 00000000..3afce5c6 --- /dev/null +++ b/packages/cultcache-ts/test/reference-numeric-table.ts @@ -0,0 +1,176 @@ +// Oracle for the numeric leaves, written by the C# reference. Each READ row is a one-slot record [value] fed to a C# +// MessagePackSerializer.Deserialize of a class with one int, long, float or double field: ACCEPT (float +// values as the double's IEEE bits) or REFUSE. Each WRITE row is the bytes this runtime writes for a value, which +// the C# reader read and re-serialized to the same bytes. Produced by a job-only probe of src/ at this commit's +// parent range; the C# test project owns no numeric vector. +export const referenceReads: readonly (readonly [leaf: string, value: string, outcome: string])[] = [ + ["int32", "05", "ACCEPT 5"], + ["int32", "ff", "ACCEPT -1"], + ["int32", "ccc8", "ACCEPT 200"], + ["int32", "cd0100", "ACCEPT 256"], + ["int32", "ceb2d05e00", "REFUSE"], + ["int32", "d0ce", "ACCEPT -50"], + ["int32", "d18000", "ACCEPT -32768"], + ["int32", "d280000000", "ACCEPT -2147483648"], + ["int32", "d38000000000000000", "REFUSE"], + ["int32", "cf0020000000000001", "REFUSE"], + ["int32", "cfffffffffffffffff", "REFUSE"], + ["int32", "cf8000000000000000", "REFUSE"], + ["int32", "d30020000000000001", "REFUSE"], + ["int32", "d3fffffffffffffff0", "ACCEPT -16"], + ["int32", "ca3fc00000", "REFUSE"], + ["int32", "cb3ff8000000000000", "REFUSE"], + ["int32", "cb401c000000000000", "REFUSE"], + ["int32", "cb7ff8000000000000", "REFUSE"], + ["int32", "ca7f800000", "REFUSE"], + ["int32", "cb7e37e43c8800759c", "REFUSE"], + ["int32", "cb47efffffe0000000", "REFUSE"], + ["int32", "cb3fb999999999999a", "REFUSE"], + ["int32", "ca3dcccccd", "REFUSE"], + ["int32", "cb8000000000000000", "REFUSE"], + ["int32", "c0", "REFUSE"], + ["int32", "c3", "REFUSE"], + ["int32", "a161", "REFUSE"], + ["int32", "c40100", "REFUSE"], + ["int32", "90", "REFUSE"], + ["int32", "80", "REFUSE"], + ["int64", "05", "ACCEPT 5"], + ["int64", "ff", "ACCEPT -1"], + ["int64", "ccc8", "ACCEPT 200"], + ["int64", "cd0100", "ACCEPT 256"], + ["int64", "ceb2d05e00", "ACCEPT 3000000000"], + ["int64", "d0ce", "ACCEPT -50"], + ["int64", "d18000", "ACCEPT -32768"], + ["int64", "d280000000", "ACCEPT -2147483648"], + ["int64", "d38000000000000000", "ACCEPT -9223372036854775808"], + ["int64", "cf0020000000000001", "ACCEPT 9007199254740993"], + ["int64", "cfffffffffffffffff", "REFUSE"], + ["int64", "cf8000000000000000", "REFUSE"], + ["int64", "d30020000000000001", "ACCEPT 9007199254740993"], + ["int64", "d3fffffffffffffff0", "ACCEPT -16"], + ["int64", "ca3fc00000", "REFUSE"], + ["int64", "cb3ff8000000000000", "REFUSE"], + ["int64", "cb401c000000000000", "REFUSE"], + ["int64", "cb7ff8000000000000", "REFUSE"], + ["int64", "ca7f800000", "REFUSE"], + ["int64", "cb7e37e43c8800759c", "REFUSE"], + ["int64", "cb47efffffe0000000", "REFUSE"], + ["int64", "cb3fb999999999999a", "REFUSE"], + ["int64", "ca3dcccccd", "REFUSE"], + ["int64", "cb8000000000000000", "REFUSE"], + ["int64", "c0", "REFUSE"], + ["int64", "c3", "REFUSE"], + ["int64", "a161", "REFUSE"], + ["int64", "c40100", "REFUSE"], + ["int64", "90", "REFUSE"], + ["int64", "80", "REFUSE"], + ["float32", "05", "ACCEPT 4014000000000000"], + ["float32", "ff", "ACCEPT bff0000000000000"], + ["float32", "ccc8", "ACCEPT 4069000000000000"], + ["float32", "cd0100", "ACCEPT 4070000000000000"], + ["float32", "ceb2d05e00", "ACCEPT 41e65a0bc0000000"], + ["float32", "d0ce", "ACCEPT c049000000000000"], + ["float32", "d18000", "ACCEPT c0e0000000000000"], + ["float32", "d280000000", "ACCEPT c1e0000000000000"], + ["float32", "d38000000000000000", "ACCEPT c3e0000000000000"], + ["float32", "cf0020000000000001", "ACCEPT 4340000000000000"], + ["float32", "cfffffffffffffffff", "ACCEPT 43f0000000000000"], + ["float32", "cf8000000000000000", "ACCEPT 43e0000000000000"], + ["float32", "d30020000000000001", "ACCEPT 4340000000000000"], + ["float32", "d3fffffffffffffff0", "ACCEPT c030000000000000"], + ["float32", "ca3fc00000", "ACCEPT 3ff8000000000000"], + ["float32", "cb3ff8000000000000", "ACCEPT 3ff8000000000000"], + ["float32", "cb401c000000000000", "ACCEPT 401c000000000000"], + ["float32", "cb7ff8000000000000", "ACCEPT NaN"], + ["float32", "ca7f800000", "ACCEPT 7ff0000000000000"], + ["float32", "cb7e37e43c8800759c", "ACCEPT 7ff0000000000000"], + ["float32", "cb47efffffe0000000", "ACCEPT 47efffffe0000000"], + ["float32", "cb3fb999999999999a", "ACCEPT 3fb99999a0000000"], + ["float32", "ca3dcccccd", "ACCEPT 3fb99999a0000000"], + ["float32", "cb8000000000000000", "ACCEPT 8000000000000000"], + ["float32", "c0", "REFUSE"], + ["float32", "c3", "REFUSE"], + ["float32", "a161", "REFUSE"], + ["float32", "c40100", "REFUSE"], + ["float32", "90", "REFUSE"], + ["float32", "80", "REFUSE"], + ["float64", "05", "ACCEPT 4014000000000000"], + ["float64", "ff", "ACCEPT bff0000000000000"], + ["float64", "ccc8", "ACCEPT 4069000000000000"], + ["float64", "cd0100", "ACCEPT 4070000000000000"], + ["float64", "ceb2d05e00", "ACCEPT 41e65a0bc0000000"], + ["float64", "d0ce", "ACCEPT c049000000000000"], + ["float64", "d18000", "ACCEPT c0e0000000000000"], + ["float64", "d280000000", "ACCEPT c1e0000000000000"], + ["float64", "d38000000000000000", "ACCEPT c3e0000000000000"], + ["float64", "cf0020000000000001", "ACCEPT 4340000000000000"], + ["float64", "cfffffffffffffffff", "ACCEPT 43f0000000000000"], + ["float64", "cf8000000000000000", "ACCEPT 43e0000000000000"], + ["float64", "d30020000000000001", "ACCEPT 4340000000000000"], + ["float64", "d3fffffffffffffff0", "ACCEPT c030000000000000"], + ["float64", "ca3fc00000", "ACCEPT 3ff8000000000000"], + ["float64", "cb3ff8000000000000", "ACCEPT 3ff8000000000000"], + ["float64", "cb401c000000000000", "ACCEPT 401c000000000000"], + ["float64", "cb7ff8000000000000", "ACCEPT NaN"], + ["float64", "ca7f800000", "ACCEPT 7ff0000000000000"], + ["float64", "cb7e37e43c8800759c", "ACCEPT 7e37e43c8800759c"], + ["float64", "cb47efffffe0000000", "ACCEPT 47efffffe0000000"], + ["float64", "cb3fb999999999999a", "ACCEPT 3fb999999999999a"], + ["float64", "ca3dcccccd", "ACCEPT 3fb99999a0000000"], + ["float64", "cb8000000000000000", "ACCEPT 8000000000000000"], + ["float64", "c0", "REFUSE"], + ["float64", "c3", "REFUSE"], + ["float64", "a161", "REFUSE"], + ["float64", "c40100", "REFUSE"], + ["float64", "90", "REFUSE"], + ["float64", "80", "REFUSE"], +]; + +export const referenceWrites: readonly (readonly [leaf: string, value: string, hex: string])[] = [ + ["int32", "0", "9100"], + ["int32", "1", "9101"], + ["int32", "127", "917f"], + ["int32", "128", "91cc80"], + ["int32", "255", "91ccff"], + ["int32", "256", "91cd0100"], + ["int32", "65535", "91cdffff"], + ["int32", "65536", "91ce00010000"], + ["int32", "-1", "91ff"], + ["int32", "-32", "91e0"], + ["int32", "-33", "91d0df"], + ["int32", "-128", "91d080"], + ["int32", "-129", "91d1ff7f"], + ["int32", "-32768", "91d18000"], + ["int32", "-32769", "91d2ffff7fff"], + ["int32", "2147483647", "91ce7fffffff"], + ["int32", "-2147483648", "91d280000000"], + ["int64", "0", "9100"], + ["int64", "127", "917f"], + ["int64", "128", "91cc80"], + ["int64", "4294967295", "91ceffffffff"], + ["int64", "4294967296", "91cf0000000100000000"], + ["int64", "9007199254740991", "91cf001fffffffffffff"], + ["int64", "-9007199254740991", "91d3ffe0000000000001"], + ["int64", "-2147483649", "91d3ffffffff7fffffff"], + ["int64", "9007199254740994n", "91cf0020000000000002"], + ["int64", "9223372036854775807n", "91cf7fffffffffffffff"], + ["int64", "-9223372036854775808n", "91d38000000000000000"], + ["int64", "9007199254740993n", "91cf0020000000000001"], + ["int64", "-9007199254740993n", "91d3ffdfffffffffffff"], + ["float32", "1.5", "91ca3fc00000"], + ["float32", "0.1", "91ca3dcccccd"], + ["float32", "0", "91ca00000000"], + ["float32", "-0", "91ca80000000"], + ["float32", "3", "91ca40400000"], + ["float32", "1e+30", "91ca7149f2ca"], + ["float32", "Infinity", "91ca7f800000"], + ["float32", "NaN", "91ca7fc00000"], + ["float64", "1.5", "91cb3ff8000000000000"], + ["float64", "0.1", "91cb3fb999999999999a"], + ["float64", "0", "91cb0000000000000000"], + ["float64", "-0", "91cb8000000000000000"], + ["float64", "3", "91cb4008000000000000"], + ["float64", "1e+300", "91cb7e37e43c8800759c"], + ["float64", "Infinity", "91cb7ff0000000000000"], + ["float64", "NaN", "91cb7ff8000000000000"], +]; diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index ab75c000..e232872f 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -12,9 +12,10 @@ import { codecs } from "../src"; import type { Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; +import { referenceReads, referenceWrites } from "./reference-numeric-table"; import { defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; -import type { CacheBackingStore, CultCacheEnvelope } from "../src/types"; +import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; const { array, boolean, bytes, float32, float64, int32, int64, lazy, list, nullable, slots, string, union, unit } = codecs; @@ -504,10 +505,10 @@ test("a codec and a formatter are refused together wherever the definition enter }); test("a definition that skips defineDocumentType still writes its codec's bytes and its derived catalog", async () => { - const raw = { type: "tests.union_surface", schemaId: "tests.union_surface.v1", schema: surfaceSchema, codec: surfaceCodec, name: "id" as const }; + const raw = { type: "tests.union_surface", schemaId: "tests.union_surface.v1", schema: surfaceSchema, codec: surfaceCodec, name: "id" } as unknown as AnyCultCacheDocumentDefinition; const holder = memoryStore([]); const cache = CultCache.builder().withDocumentType(raw).withGenericStore(holder).build(); - await cache.put(raw, "surface", golden); + await cache.put(raw as never, "surface", golden as never); assert.equal(hex(holder.entries[0].payload), hex(await read("union-surface.msgpack"))); const [snapshot] = cache.snapshot(); assert.deepEqual(snapshot.catalogEntry!.members!.map((member) => member.typeName).slice(0, 2), ["System.String", "System.Int32"]); @@ -531,6 +532,47 @@ test("every leaf and wrapper carries the reference's type name, and a catalog is assert.equal(given.members!.length, 1); }); +// referenceReads and referenceWrites are the C# reference's own answers (see the table's header): a numeric leaf reads +// and refuses exactly what the reference does, and writes exactly the bytes the reference reads back unchanged. +const numericLeaves: Record> = { int32, int64, float32, float64 }; +const bitsOf = (value: number) => (Number.isNaN(value) ? "NaN" : Buffer.from(new Float64Array([value]).buffer).reverse().toString("hex")); + +test("numeric leaves read and refuse exactly what the C# reference does", () => { + assert.equal(referenceReads.length, 120); + let accepted = 0; + let refused = 0; + for (const [leaf, value, outcome] of referenceReads) { + const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); + let got: string; + try { + const read = record.decode(readWire(fromHex(`91${value}`))).v; + got = `ACCEPT ${leaf.startsWith("float") ? bitsOf(read as number) : String(read)}`; + accepted += 1; + } catch (error) { + assert.ok(!(error as Error).message.includes(value), "a refusal never echoes the value"); + got = "REFUSE"; + refused += 1; + } + assert.equal(got, outcome, `${leaf} ${value}`); + } + assert.deepEqual([accepted, refused], [referenceReads.filter((row) => row[2].startsWith("ACCEPT")).length, referenceReads.filter((row) => row[2] === "REFUSE").length]); + assert.ok(accepted > 40 && refused > 40); +}); + +test("numeric leaves write the bytes the C# reference reads back unchanged, and 64-bit values lose nothing", () => { + assert.equal(referenceWrites.length, 46); + for (const [leaf, text, expected] of referenceWrites) { + const value = leaf.startsWith("float") ? (text === "-0" ? -0 : Number(text)) : text.endsWith("n") || Math.abs(Number(text)) > Number.MAX_SAFE_INTEGER ? BigInt(text.replace("n", "")) : Number(text); + const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); + assert.equal(hex(writeWire(record.encode({ v: value }))), expected, `${leaf} ${text}`); + } + const wide = slots<{ v: number | bigint }>({ 0: ["v", int64] }); + for (const value of [2n ** 53n + 1n, 2n ** 63n - 1n, -(2n ** 63n), -(2n ** 53n) - 1n]) { + assert.equal(wide.decode(readWire(writeWire(wide.encode({ v: value })))).v, value); + } + assert.equal(wide.decode(readWire(writeWire(wide.encode({ v: 2n })))).v, 2, "a bigint that fits comes back a number"); +}); + function envelopeOf(payload: Uint8Array): CultCacheEnvelope { return { key: "surface", type: "tests.union_surface", payload, storedAt: new Date().toISOString() }; } From 0f40e4290453e6abaaa974e37892bac38361c45a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 04:56:29 +0200 Subject: [PATCH 12/44] cultcache-ts README: leaves, type names, arm keys Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 26b94ecc..06e1f3f8 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -243,6 +243,8 @@ union as `[armKey, [armSlots...]]`. A union value is `{ kind: armName, ...armFie import { codecs, defineDocumentType } from "@gamecult/cultcache-ts"; const { slots, union, list, lazy, string } = codecs; +const NS = "MyApp.Scene+"; // the C# names of the types, as the reference writes them into a store's schema catalog + type Node = | { kind: "Box"; id: string; children: (Node | null)[] } | { kind: "Text"; id: string; content: string } @@ -252,7 +254,7 @@ const node: codecs.Codec = union("Node", { 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })], 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })], 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], // slot 2 retired -}); +}, `${NS}Node`); const sceneDocument = defineDocumentType({ type: "scene", @@ -267,6 +269,16 @@ const sceneDocument = defineDocumentType({ - A union refuses an undeclared arm key on read, naming the union, the key and the declared keys. Nothing falls back to a default arm. An arm with no slots is `codecs.unit` and is written `[key, []]`. - A refusal names the union, the field and the slot; it never echoes the value it refused. +- The leaves are `string`, `boolean`, `bytes`, `int32`, `int64`, `float32` and `float64`, each writing the MessagePack the + C# type writes and refusing what C# refuses (a float in an int slot, an `int` outside Int32). `int64` reads a number + when the value is a safe integer and a bigint beyond 2^53, and refuses a number beyond 2^53 on write. `list`, `array` + and `nullable` wrap a leaf, a union or a record. +- Every codec carries the reference's type name, and a store's schema catalog is derived from those names, because the + reference refuses a store whose slot names another type. The leaves know theirs; give a union or a record its C# name + as the last argument of `union(name, arms, typeName)` or `slots(spec, typeName)`. A codec with a slot it cannot name + refuses to derive members: pass `members` instead. +- An arm key is an integer within Int32, in any MessagePack integer encoding. A float key (even `1.0`) is refused on + read, and a union refuses a key outside Int32 when it is defined. - `codec` supplies the document's formatter and, unless `members` is given, its catalog members. A definition may not carry both `codec` and `formatter`. `schema.parse` still runs on every decoded value. From 0882eee9d2ad8fed0f855ab1206a89e9a5903a36 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 05:04:23 +0200 Subject: [PATCH 13/44] cultcache-ts wire: delete the dead guards the mutation run found; pin the width and Int32 boundaries Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/wire.ts | 5 +- .../cultcache-ts/test/union-parity.test.ts | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index d57ff049..d9102237 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -48,7 +48,7 @@ function putLength(length: number, parts: number[], fix: number, fixMax: number, } function put(node: unknown, parts: number[]): void { - if (node === null || node === undefined) parts.push(0xc0); + if (node === null) parts.push(0xc0); else if (typeof node === "boolean") parts.push(node ? 0xc3 : 0xc2); else if (typeof node === "bigint") putInt(node, parts); else if (typeof node === "number") { @@ -78,7 +78,7 @@ export function readWire(payload: Uint8Array): unknown { let offset = 0; const take = (count: number): number => { - if (count < 0 || offset + count > payload.length) throw new Error("the MessagePack payload is truncated."); + if (offset + count > payload.length) throw new Error("the MessagePack payload is truncated."); offset += count; return offset - count; }; @@ -86,7 +86,6 @@ export function readWire(payload: Uint8Array): unknown { const string = (length: number): string => utf8Decoder.decode(payload.subarray(take(length), offset)); const binary = (length: number): Uint8Array => payload.slice(take(length), offset); const array = (length: number): unknown[] => { - if (length > payload.length - offset) throw new Error("the MessagePack payload is truncated."); const out: unknown[] = []; for (let index = 0; index < length; index += 1) out.push(next()); return out; diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index e232872f..cb76ed03 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -573,6 +573,63 @@ test("numeric leaves write the bytes the C# reference reads back unchanged, and assert.equal(wide.decode(readWire(writeWire(wide.encode({ v: 2n })))).v, 2, "a bigint that fits comes back a number"); }); +test("the wire writer's width boundaries: where a length or an integer changes its header", () => { + assert.equal(hex(writeWire(0)), "00"); + assert.equal(hex(writeWire("a".repeat(31))).slice(0, 2), "bf"); + assert.equal(hex(writeWire("a".repeat(32))).slice(0, 4), "d920"); + assert.equal(hex(writeWire("a".repeat(255))).slice(0, 4), "d9ff"); + assert.equal(hex(writeWire("a".repeat(256))).slice(0, 6), "da0100"); + assert.equal(hex(writeWire("a".repeat(65535))).slice(0, 6), "daffff"); + assert.equal(hex(writeWire("a".repeat(65536))).slice(0, 10), "db00010000"); + assert.equal(hex(writeWire(new Uint8Array(1))), "c40100"); + assert.equal(hex(writeWire(new Uint8Array(256))).slice(0, 6), "c50100"); + assert.equal(hex(writeWire(new Uint8Array(65536))).slice(0, 10), "c600010000"); + assert.equal(hex(writeWire(new Array(15).fill(null))), `9f${"c0".repeat(15)}`); + assert.equal(hex(writeWire(new Array(65535).fill(null))).slice(0, 6), "dcffff"); + assert.equal(hex(writeWire(new Array(65536).fill(null))).slice(0, 10), "dd00010000"); + assert.throws(() => writeWire(-(2n ** 63n) - 1n), { message: "cannot write an integer wider than 64 bits." }); + assert.throws(() => writeWire(undefined), /this kind of value/u); + assert.equal(hex(writeWire(-(2n ** 63n))), "d38000000000000000"); +}); + +test("the wire reader's boundaries: the largest fixarray, the 32-bit binary and string, the safe integer edge", () => { + assert.deepEqual(readWire(fromHex(`9f${"c0".repeat(15)}`)), new Array(15).fill(null)); + assert.deepEqual(readWire(fromHex("c600000001aa")), new Uint8Array([0xaa])); + assert.equal(readWire(fromHex("db0000000161")), "a"); + assert.equal(readWire(fromHex("d3ffe0000000000001")), -(2 ** 53 - 1)); + assert.equal(readWire(fromHex("d3ffe0000000000000")), -(2n ** 53n)); + assert.equal(readWire(fromHex("cf0020000000000000")), 2n ** 53n); +}); + +test("a refusal says what was found: nil, a float, an integer that is not the slot's, a plain kind", () => { + assert.throws(() => int32.encode(undefined as unknown as number), { message: "expected an int32, found nil" }); + assert.throws(() => string.decode(new WireFloat(1, true)), { message: "expected a string, found float" }); + assert.throws(() => string.decode(5n), { message: "expected a string, found integer" }); + assert.throws(() => int32.decode(new WireFloat(7, true)), { message: "expected an int32, found float" }); + assert.throws(() => float64.decode("1"), { message: "expected a float64, found string" }); + assert.throws(() => int64.encode(1.5), { message: "expected an int64, found a non-integer number" }); +}); + +test("nullable names a value type Nullable and anything else as itself", () => { + assert.equal(nullable(boolean).typeName, "System.Nullable"); + assert.equal(nullable(int64).typeName, "System.Nullable"); + assert.equal(nullable(float32).typeName, "System.Nullable"); + assert.equal(nullable(float64).typeName, "System.Nullable"); + assert.equal(nullable(bytes).typeName, "System.Byte[]"); + const unnamedValueType: Codec = { valueType: true, encode: (value) => value, decode: (raw) => raw as number }; + assert.equal(nullable(unnamedValueType).typeName, undefined); + assert.equal(lazy(() => int32).valueType, true); +}); + +test("the Int32 edges of an arm key: the largest is an arm, one past either end is outside", () => { + const edge = union<{ kind: "A" }>("U", { 2147483647: ["A", unit] }); + assert.deepEqual(edge.decode(readWire(fromHex("92ce7fffffff90"))), { kind: "A" }); + assert.deepEqual(edge.encode({ kind: "A" }), [2147483647, []]); + assert.throws(() => edge.decode(readWire(fromHex("92ce8000000090"))), { message: "Union U: the arm key is outside Int32 (declared keys: 2147483647)." }); + assert.throws(() => edge.decode(readWire(fromHex("92d3ffffffff7fffffff90"))), { message: "Union U: the arm key is outside Int32 (declared keys: 2147483647)." }); + assert.throws(() => edge.decode(readWire(fromHex("92d3ffffffff8000000090"))), { message: "Union U: unknown arm key -2147483648 (declared keys: 2147483647)." }); +}); + function envelopeOf(payload: Uint8Array): CultCacheEnvelope { return { key: "surface", type: "tests.union_surface", payload, storedAt: new Date().toISOString() }; } From c38974f3c6d74de216b2b5c5ab9ecc4bdaa777cc Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:30:04 +0200 Subject: [PATCH 14/44] C# numeric-leaf vector writer: reads and writes of int, long, float, double slots, gated by CULTCACHE_WRITE_VECTORS The vector files follow in the next commit; the new tests fail until they are committed. Co-Authored-By: Claude Opus 5.5 --- .../NumericLeafVectorTests.cs | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs new file mode 100644 index 00000000..c7233a8a --- /dev/null +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -0,0 +1,223 @@ +#nullable enable +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Globalization; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The numeric-leaf golden in tests/vectors/numeric-leaves: what the reference reads and refuses for an int, long, float + // and double slot, and the bytes it writes for them. The C# reference is the only writer (CULTCACHE_WRITE_VECTORS=1); + // every other run reads the committed bytes and compares them byte for byte. Other runtimes are held to these files. + public class NumericLeafVectorTests + { + private const string WriteVariable = "CULTCACHE_WRITE_VECTORS"; + private static readonly string[] Leaves = { "int32", "int64", "float32", "float64" }; + + [MessagePackObject] public sealed class Int32Slot { [Key(0)] public int V; } + [MessagePackObject] public sealed class Int64Slot { [Key(0)] public long V; } + [MessagePackObject] public sealed class Float32Slot { [Key(0)] public float V; } + [MessagePackObject] public sealed class Float64Slot { [Key(0)] public double V; } + + private static bool Writing => Environment.GetEnvironmentVariable(WriteVariable) == "1"; + + private static readonly MessagePackSerializerOptions Options = CultDocumentMessagePackSerialization.OptionsFor(typeof(NumericLeafVectorTests).Assembly); + + private static string VectorPath(string name) + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var vectors = Path.Combine(dir.FullName, "tests", "vectors"); + if (Directory.Exists(vectors)) return Path.Combine(vectors, "numeric-leaves", name); + } + + throw new DirectoryNotFoundException($"tests/vectors not found above {TestContext.CurrentContext.TestDirectory}."); + } + + // Under the write variable the reference's bytes become the file; otherwise they must equal the committed file. + private static void Check(string name, byte[] reference) + { + var path = VectorPath(name); + if (Writing) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, reference); + return; + } + + Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); + Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}"); + } + + // ---- reads: a one-slot record [value], every value fed to every leaf ---- + + // The value token of each read case, as MessagePack hex: integers of every width and edge, floats, and kinds no leaf reads. + private static readonly string[] ReadValues = + { + "05", "ff", "ccc8", "cd0100", "ceb2d05e00", "d0ce", "d18000", "d280000000", "d38000000000000000", + "cf0020000000000001", "cfffffffffffffffff", "cf8000000000000000", "d30020000000000001", "d3fffffffffffffff0", + // integers beyond 2^53 that a float32 slot must round once: past a tie, on a tie to even, on a tie to odd, negative + "d31000001000000001", "cf8000008000000001", "cf8000008000000000", "cf8000018000000000", "d3efffffe800000001", "d3effffff000000001", + "ca3fc00000", "cb3ff8000000000000", "cb401c000000000000", "cb7ff8000000000000", "ca7f800000", "ca7f800001", "cb7e37e43c8800759c", + "cb47efffffe0000000", "cb3fb999999999999a", "ca3dcccccd", "cb8000000000000000", + "c0", "c3", "a161", "c40100", "90", "80", + }; + + private static (bool Accepted, byte[] Bits) Read(string leaf, byte[] payload) + { + try + { + var bits = new byte[8]; + switch (leaf) + { + case "int32": BinaryPrimitives.WriteInt64BigEndian(bits, MessagePackSerializer.Deserialize(payload, Options).V); break; + case "int64": BinaryPrimitives.WriteInt64BigEndian(bits, MessagePackSerializer.Deserialize(payload, Options).V); break; + case "float32": BinaryPrimitives.WriteInt64BigEndian(bits, BitConverter.DoubleToInt64Bits(MessagePackSerializer.Deserialize(payload, Options).V)); break; + default: BinaryPrimitives.WriteInt64BigEndian(bits, BitConverter.DoubleToInt64Bits(MessagePackSerializer.Deserialize(payload, Options).V)); break; + } + + return (true, bits); + } + catch (Exception) + { + return (false, Array.Empty()); + } + } + + private static byte[] Payload(string valueHex) => Convert.FromHexString("91" + valueHex); + + // reads.msgpack: an array of [leaf, payload (bin), accepted, bits (bin of 8: the int64 or the double's IEEE bits, big-endian; empty when refused)]. + private static byte[] ReadsVector() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(Leaves.Length * ReadValues.Length); + foreach (var leaf in Leaves) + foreach (var value in ReadValues) + { + var payload = Payload(value); + var (accepted, bits) = Read(leaf, payload); + writer.WriteArrayHeader(4); + writer.Write(leaf); + writer.Write(payload); + writer.Write(accepted); + writer.Write(bits); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + // ---- writes: the reference's own bytes for a value ---- + + // text is a decimal for an integer leaf and the double's IEEE bits (hex) for a float leaf. + private static readonly (string Leaf, string Text)[] WriteValues = + { + ("int32", "0"), ("int32", "1"), ("int32", "127"), ("int32", "128"), ("int32", "255"), ("int32", "256"), ("int32", "65535"), ("int32", "65536"), + ("int32", "-1"), ("int32", "-32"), ("int32", "-33"), ("int32", "-128"), ("int32", "-129"), ("int32", "-32768"), ("int32", "-32769"), + ("int32", "2147483647"), ("int32", "-2147483648"), + ("int64", "0"), ("int64", "127"), ("int64", "128"), ("int64", "4294967295"), ("int64", "4294967296"), ("int64", "9007199254740991"), + ("int64", "-9007199254740991"), ("int64", "-2147483649"), ("int64", "9007199254740994"), ("int64", "9223372036854775807"), + ("int64", "-9223372036854775808"), ("int64", "9007199254740993"), ("int64", "-9007199254740993"), + ("float32", "3ff8000000000000"), ("float32", "3fb999999999999a"), ("float32", "0000000000000000"), ("float32", "8000000000000000"), + ("float32", "4008000000000000"), ("float32", "46293e5939a08cea"), ("float32", "7ff0000000000000"), ("float32", "7ff8000000000000"), + ("float64", "3ff8000000000000"), ("float64", "3fb999999999999a"), ("float64", "0000000000000000"), ("float64", "8000000000000000"), + ("float64", "4008000000000000"), ("float64", "7e37e43c8800759c"), ("float64", "7ff0000000000000"), ("float64", "7ff8000000000000"), + }; + + private static byte[] Write(string leaf, string text) + { + double Double() => BitConverter.Int64BitsToDouble((long)ulong.Parse(text, NumberStyles.HexNumber, CultureInfo.InvariantCulture)); + return leaf switch + { + "int32" => MessagePackSerializer.Serialize(new Int32Slot { V = int.Parse(text, CultureInfo.InvariantCulture) }, Options), + "int64" => MessagePackSerializer.Serialize(new Int64Slot { V = long.Parse(text, CultureInfo.InvariantCulture) }, Options), + "float32" => MessagePackSerializer.Serialize(new Float32Slot { V = (float)Double() }, Options), + _ => MessagePackSerializer.Serialize(new Float64Slot { V = Double() }, Options), + }; + } + + // writes.msgpack: an array of [leaf, text, bytes (bin): the one-slot record the reference writes for the value]. + private static byte[] WritesVector() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(WriteValues.Length); + foreach (var (leaf, text) in WriteValues) + { + writer.WriteArrayHeader(3); + writer.Write(leaf); + writer.Write(text); + writer.Write(Write(leaf, text)); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + // ---- tests ---- + + [Test] + public void ReadsVectorMatches() => Check("reads.msgpack", ReadsVector()); + + [Test] + public void WritesVectorMatches() => Check("writes.msgpack", WritesVector()); + + [Test] + public void ReadsCoverEveryLeafAndBothOutcomes() + { + var rows = Leaves.SelectMany(leaf => ReadValues.Select(value => (leaf, value, result: Read(leaf, Payload(value))))).ToList(); + Assert.That(rows, Has.Count.EqualTo(Leaves.Length * ReadValues.Length)); + foreach (var leaf in Leaves) + { + Assert.That(rows.Count(row => row.leaf == leaf && row.result.Accepted), Is.GreaterThan(10), $"{leaf} accepts"); + Assert.That(rows.Count(row => row.leaf == leaf && !row.result.Accepted), Is.GreaterThan(5), $"{leaf} refuses"); + } + } + + // One rounding from int64 or uint64 to float, as the reference does: 2^60 + 2^36 + 1 is past the tie, so it rounds up. + [TestCase("float32", "d31000001000000001", "43b0000020000000")] + [TestCase("float32", "cf8000008000000001", "43e0000020000000")] + [TestCase("float32", "cf8000008000000000", "43e0000000000000")] + [TestCase("float32", "cf8000018000000000", "43e0000040000000")] + [TestCase("float64", "cf0020000000000001", "4340000000000000")] + [TestCase("int64", "cf0020000000000001", "0020000000000001")] + public void ReadsRoundExactlyAsTheReferenceDoes(string leaf, string value, string bitsHex) + { + var (accepted, bits) = Read(leaf, Payload(value)); + Assert.That(accepted, Is.True); + Assert.That(Convert.ToHexString(bits).ToLowerInvariant(), Is.EqualTo(bitsHex)); + } + + [Test] + public void ValuesOutsideTheLeafAreRefused() + { + Assert.That(Read("int32", Payload("ceb2d05e00")).Accepted, Is.False); + Assert.That(Read("int64", Payload("cfffffffffffffffff")).Accepted, Is.False); + Assert.That(Read("int32", Payload("ca3fc00000")).Accepted, Is.False); + } + + [Test] + public void EveryWrittenValueIsReadBackAndWrittenAgainUnchanged() + { + foreach (var (leaf, text) in WriteValues) + { + var written = Write(leaf, text); + Assert.That(Read(leaf, written).Accepted, Is.True, $"{leaf} {text}"); + var again = leaf switch + { + "int32" => MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(written, Options), Options), + "int64" => MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(written, Options), Options), + "float32" => MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(written, Options), Options), + _ => MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(written, Options), Options), + }; + Assert.That(Convert.ToHexString(again), Is.EqualTo(Convert.ToHexString(written)), $"{leaf} {text}"); + } + } + } +} From 42f746de201bc121e4b1822af51b5199c1790cf5 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:30:48 +0200 Subject: [PATCH 15/44] NumericLeafVectorTests: the coverage floor matches the cases (int32 accepts 8) Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index c7233a8a..b270947c 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -175,7 +175,7 @@ namespace GameCult.Caching.Tests Assert.That(rows, Has.Count.EqualTo(Leaves.Length * ReadValues.Length)); foreach (var leaf in Leaves) { - Assert.That(rows.Count(row => row.leaf == leaf && row.result.Accepted), Is.GreaterThan(10), $"{leaf} accepts"); + Assert.That(rows.Count(row => row.leaf == leaf && row.result.Accepted), Is.GreaterThan(5), $"{leaf} accepts"); Assert.That(rows.Count(row => row.leaf == leaf && !row.result.Accepted), Is.GreaterThan(5), $"{leaf} refuses"); } } From 3eb4cb641fb443eb2fabbd5de392d79d09587ed0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:32:18 +0200 Subject: [PATCH 16/44] NumericLeafVectorTests: a mixed-record vector, the shape a replicated document has Co-Authored-By: Claude Opus 5.5 --- .../test/reference-numeric-table.ts | 176 ------------------ .../NumericLeafVectorTests.cs | 24 +++ 2 files changed, 24 insertions(+), 176 deletions(-) delete mode 100644 packages/cultcache-ts/test/reference-numeric-table.ts diff --git a/packages/cultcache-ts/test/reference-numeric-table.ts b/packages/cultcache-ts/test/reference-numeric-table.ts deleted file mode 100644 index 3afce5c6..00000000 --- a/packages/cultcache-ts/test/reference-numeric-table.ts +++ /dev/null @@ -1,176 +0,0 @@ -// Oracle for the numeric leaves, written by the C# reference. Each READ row is a one-slot record [value] fed to a C# -// MessagePackSerializer.Deserialize of a class with one int, long, float or double field: ACCEPT (float -// values as the double's IEEE bits) or REFUSE. Each WRITE row is the bytes this runtime writes for a value, which -// the C# reader read and re-serialized to the same bytes. Produced by a job-only probe of src/ at this commit's -// parent range; the C# test project owns no numeric vector. -export const referenceReads: readonly (readonly [leaf: string, value: string, outcome: string])[] = [ - ["int32", "05", "ACCEPT 5"], - ["int32", "ff", "ACCEPT -1"], - ["int32", "ccc8", "ACCEPT 200"], - ["int32", "cd0100", "ACCEPT 256"], - ["int32", "ceb2d05e00", "REFUSE"], - ["int32", "d0ce", "ACCEPT -50"], - ["int32", "d18000", "ACCEPT -32768"], - ["int32", "d280000000", "ACCEPT -2147483648"], - ["int32", "d38000000000000000", "REFUSE"], - ["int32", "cf0020000000000001", "REFUSE"], - ["int32", "cfffffffffffffffff", "REFUSE"], - ["int32", "cf8000000000000000", "REFUSE"], - ["int32", "d30020000000000001", "REFUSE"], - ["int32", "d3fffffffffffffff0", "ACCEPT -16"], - ["int32", "ca3fc00000", "REFUSE"], - ["int32", "cb3ff8000000000000", "REFUSE"], - ["int32", "cb401c000000000000", "REFUSE"], - ["int32", "cb7ff8000000000000", "REFUSE"], - ["int32", "ca7f800000", "REFUSE"], - ["int32", "cb7e37e43c8800759c", "REFUSE"], - ["int32", "cb47efffffe0000000", "REFUSE"], - ["int32", "cb3fb999999999999a", "REFUSE"], - ["int32", "ca3dcccccd", "REFUSE"], - ["int32", "cb8000000000000000", "REFUSE"], - ["int32", "c0", "REFUSE"], - ["int32", "c3", "REFUSE"], - ["int32", "a161", "REFUSE"], - ["int32", "c40100", "REFUSE"], - ["int32", "90", "REFUSE"], - ["int32", "80", "REFUSE"], - ["int64", "05", "ACCEPT 5"], - ["int64", "ff", "ACCEPT -1"], - ["int64", "ccc8", "ACCEPT 200"], - ["int64", "cd0100", "ACCEPT 256"], - ["int64", "ceb2d05e00", "ACCEPT 3000000000"], - ["int64", "d0ce", "ACCEPT -50"], - ["int64", "d18000", "ACCEPT -32768"], - ["int64", "d280000000", "ACCEPT -2147483648"], - ["int64", "d38000000000000000", "ACCEPT -9223372036854775808"], - ["int64", "cf0020000000000001", "ACCEPT 9007199254740993"], - ["int64", "cfffffffffffffffff", "REFUSE"], - ["int64", "cf8000000000000000", "REFUSE"], - ["int64", "d30020000000000001", "ACCEPT 9007199254740993"], - ["int64", "d3fffffffffffffff0", "ACCEPT -16"], - ["int64", "ca3fc00000", "REFUSE"], - ["int64", "cb3ff8000000000000", "REFUSE"], - ["int64", "cb401c000000000000", "REFUSE"], - ["int64", "cb7ff8000000000000", "REFUSE"], - ["int64", "ca7f800000", "REFUSE"], - ["int64", "cb7e37e43c8800759c", "REFUSE"], - ["int64", "cb47efffffe0000000", "REFUSE"], - ["int64", "cb3fb999999999999a", "REFUSE"], - ["int64", "ca3dcccccd", "REFUSE"], - ["int64", "cb8000000000000000", "REFUSE"], - ["int64", "c0", "REFUSE"], - ["int64", "c3", "REFUSE"], - ["int64", "a161", "REFUSE"], - ["int64", "c40100", "REFUSE"], - ["int64", "90", "REFUSE"], - ["int64", "80", "REFUSE"], - ["float32", "05", "ACCEPT 4014000000000000"], - ["float32", "ff", "ACCEPT bff0000000000000"], - ["float32", "ccc8", "ACCEPT 4069000000000000"], - ["float32", "cd0100", "ACCEPT 4070000000000000"], - ["float32", "ceb2d05e00", "ACCEPT 41e65a0bc0000000"], - ["float32", "d0ce", "ACCEPT c049000000000000"], - ["float32", "d18000", "ACCEPT c0e0000000000000"], - ["float32", "d280000000", "ACCEPT c1e0000000000000"], - ["float32", "d38000000000000000", "ACCEPT c3e0000000000000"], - ["float32", "cf0020000000000001", "ACCEPT 4340000000000000"], - ["float32", "cfffffffffffffffff", "ACCEPT 43f0000000000000"], - ["float32", "cf8000000000000000", "ACCEPT 43e0000000000000"], - ["float32", "d30020000000000001", "ACCEPT 4340000000000000"], - ["float32", "d3fffffffffffffff0", "ACCEPT c030000000000000"], - ["float32", "ca3fc00000", "ACCEPT 3ff8000000000000"], - ["float32", "cb3ff8000000000000", "ACCEPT 3ff8000000000000"], - ["float32", "cb401c000000000000", "ACCEPT 401c000000000000"], - ["float32", "cb7ff8000000000000", "ACCEPT NaN"], - ["float32", "ca7f800000", "ACCEPT 7ff0000000000000"], - ["float32", "cb7e37e43c8800759c", "ACCEPT 7ff0000000000000"], - ["float32", "cb47efffffe0000000", "ACCEPT 47efffffe0000000"], - ["float32", "cb3fb999999999999a", "ACCEPT 3fb99999a0000000"], - ["float32", "ca3dcccccd", "ACCEPT 3fb99999a0000000"], - ["float32", "cb8000000000000000", "ACCEPT 8000000000000000"], - ["float32", "c0", "REFUSE"], - ["float32", "c3", "REFUSE"], - ["float32", "a161", "REFUSE"], - ["float32", "c40100", "REFUSE"], - ["float32", "90", "REFUSE"], - ["float32", "80", "REFUSE"], - ["float64", "05", "ACCEPT 4014000000000000"], - ["float64", "ff", "ACCEPT bff0000000000000"], - ["float64", "ccc8", "ACCEPT 4069000000000000"], - ["float64", "cd0100", "ACCEPT 4070000000000000"], - ["float64", "ceb2d05e00", "ACCEPT 41e65a0bc0000000"], - ["float64", "d0ce", "ACCEPT c049000000000000"], - ["float64", "d18000", "ACCEPT c0e0000000000000"], - ["float64", "d280000000", "ACCEPT c1e0000000000000"], - ["float64", "d38000000000000000", "ACCEPT c3e0000000000000"], - ["float64", "cf0020000000000001", "ACCEPT 4340000000000000"], - ["float64", "cfffffffffffffffff", "ACCEPT 43f0000000000000"], - ["float64", "cf8000000000000000", "ACCEPT 43e0000000000000"], - ["float64", "d30020000000000001", "ACCEPT 4340000000000000"], - ["float64", "d3fffffffffffffff0", "ACCEPT c030000000000000"], - ["float64", "ca3fc00000", "ACCEPT 3ff8000000000000"], - ["float64", "cb3ff8000000000000", "ACCEPT 3ff8000000000000"], - ["float64", "cb401c000000000000", "ACCEPT 401c000000000000"], - ["float64", "cb7ff8000000000000", "ACCEPT NaN"], - ["float64", "ca7f800000", "ACCEPT 7ff0000000000000"], - ["float64", "cb7e37e43c8800759c", "ACCEPT 7e37e43c8800759c"], - ["float64", "cb47efffffe0000000", "ACCEPT 47efffffe0000000"], - ["float64", "cb3fb999999999999a", "ACCEPT 3fb999999999999a"], - ["float64", "ca3dcccccd", "ACCEPT 3fb99999a0000000"], - ["float64", "cb8000000000000000", "ACCEPT 8000000000000000"], - ["float64", "c0", "REFUSE"], - ["float64", "c3", "REFUSE"], - ["float64", "a161", "REFUSE"], - ["float64", "c40100", "REFUSE"], - ["float64", "90", "REFUSE"], - ["float64", "80", "REFUSE"], -]; - -export const referenceWrites: readonly (readonly [leaf: string, value: string, hex: string])[] = [ - ["int32", "0", "9100"], - ["int32", "1", "9101"], - ["int32", "127", "917f"], - ["int32", "128", "91cc80"], - ["int32", "255", "91ccff"], - ["int32", "256", "91cd0100"], - ["int32", "65535", "91cdffff"], - ["int32", "65536", "91ce00010000"], - ["int32", "-1", "91ff"], - ["int32", "-32", "91e0"], - ["int32", "-33", "91d0df"], - ["int32", "-128", "91d080"], - ["int32", "-129", "91d1ff7f"], - ["int32", "-32768", "91d18000"], - ["int32", "-32769", "91d2ffff7fff"], - ["int32", "2147483647", "91ce7fffffff"], - ["int32", "-2147483648", "91d280000000"], - ["int64", "0", "9100"], - ["int64", "127", "917f"], - ["int64", "128", "91cc80"], - ["int64", "4294967295", "91ceffffffff"], - ["int64", "4294967296", "91cf0000000100000000"], - ["int64", "9007199254740991", "91cf001fffffffffffff"], - ["int64", "-9007199254740991", "91d3ffe0000000000001"], - ["int64", "-2147483649", "91d3ffffffff7fffffff"], - ["int64", "9007199254740994n", "91cf0020000000000002"], - ["int64", "9223372036854775807n", "91cf7fffffffffffffff"], - ["int64", "-9223372036854775808n", "91d38000000000000000"], - ["int64", "9007199254740993n", "91cf0020000000000001"], - ["int64", "-9007199254740993n", "91d3ffdfffffffffffff"], - ["float32", "1.5", "91ca3fc00000"], - ["float32", "0.1", "91ca3dcccccd"], - ["float32", "0", "91ca00000000"], - ["float32", "-0", "91ca80000000"], - ["float32", "3", "91ca40400000"], - ["float32", "1e+30", "91ca7149f2ca"], - ["float32", "Infinity", "91ca7f800000"], - ["float32", "NaN", "91ca7fc00000"], - ["float64", "1.5", "91cb3ff8000000000000"], - ["float64", "0.1", "91cb3fb999999999999a"], - ["float64", "0", "91cb0000000000000000"], - ["float64", "-0", "91cb8000000000000000"], - ["float64", "3", "91cb4008000000000000"], - ["float64", "1e+300", "91cb7e37e43c8800759c"], - ["float64", "Infinity", "91cb7ff0000000000000"], - ["float64", "NaN", "91cb7ff8000000000000"], -]; diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index b270947c..290d433e 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -24,6 +24,17 @@ namespace GameCult.Caching.Tests [MessagePackObject] public sealed class Float32Slot { [Key(0)] public float V; } [MessagePackObject] public sealed class Float64Slot { [Key(0)] public double V; } + // A record of every leaf kind, the shape a replicated document has. + [MessagePackObject] + public sealed class Mixed + { + [Key(0)] public string Id = string.Empty; + [Key(1)] public int I; + [Key(2)] public long L; + [Key(3)] public float F; + [Key(4)] public double D; + } + private static bool Writing => Environment.GetEnvironmentVariable(WriteVariable) == "1"; private static readonly MessagePackSerializerOptions Options = CultDocumentMessagePackSerialization.OptionsFor(typeof(NumericLeafVectorTests).Assembly); @@ -160,6 +171,9 @@ namespace GameCult.Caching.Tests return buffer.WrittenSpan.ToArray(); } + // mixed-record.msgpack: the reference's bytes for Mixed { "n1", 7, 8, 1.5f, 2.5 }, which other runtimes must write identically. + private static byte[] MixedRecord() => MessagePackSerializer.Serialize(new Mixed { Id = "n1", I = 7, L = 8, F = 1.5f, D = 2.5 }, Options); + // ---- tests ---- [Test] @@ -168,6 +182,16 @@ namespace GameCult.Caching.Tests [Test] public void WritesVectorMatches() => Check("writes.msgpack", WritesVector()); + [Test] + public void MixedRecordVectorMatches() + { + var written = MixedRecord(); + Assert.That(Convert.ToHexString(written), Is.EqualTo("95A26E310708CA3FC00000CB4004000000000000")); + Check("mixed-record.msgpack", written); + var read = MessagePackSerializer.Deserialize(written, Options); + Assert.That((read.Id, read.I, read.L, read.F, read.D), Is.EqualTo(("n1", 7, 8L, 1.5f, 2.5))); + } + [Test] public void ReadsCoverEveryLeafAndBothOutcomes() { From 35386acc51d28428d0d4093071587f422bb1c1a1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:34:30 +0200 Subject: [PATCH 17/44] numeric-leaves vectors: reads, writes and a mixed record, written by the C# reference Regenerated twice on Yggdrasil under CULTCACHE_WRITE_VECTORS=1 and byte-identical. The cultcache-ts test still imports the deleted hand table until the next commit, so this commit and its parent do not build the TS tests. Co-Authored-By: Claude Opus 5.5 --- tests/vectors/numeric-leaves/README.md | 14 ++++++++++++++ .../vectors/numeric-leaves/mixed-record.msgpack | Bin 0 -> 20 bytes tests/vectors/numeric-leaves/reads.msgpack | Bin 0 -> 3643 bytes tests/vectors/numeric-leaves/writes.msgpack | Bin 0 -> 1276 bytes 4 files changed, 14 insertions(+) create mode 100644 tests/vectors/numeric-leaves/README.md create mode 100644 tests/vectors/numeric-leaves/mixed-record.msgpack create mode 100644 tests/vectors/numeric-leaves/reads.msgpack create mode 100644 tests/vectors/numeric-leaves/writes.msgpack diff --git a/tests/vectors/numeric-leaves/README.md b/tests/vectors/numeric-leaves/README.md new file mode 100644 index 00000000..e7ecb070 --- /dev/null +++ b/tests/vectors/numeric-leaves/README.md @@ -0,0 +1,14 @@ +# numeric-leaves vectors + +Written only by the C# reference, `tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs`, with +`CULTCACHE_WRITE_VECTORS=1`. Every other run reads these files and compares them byte for byte; a changed vector is a +reviewed commit. Other runtimes decode them and are held to the same answers. + +A document with one slot, `[value]`, whose slot is a C# `int`, `long`, `float` or `double` (leaf `int32`, `int64`, +`float32`, `float64`). + +- `reads.msgpack`: an array of `[leaf, payload, accepted, bits]`. `payload` (bin) is the one-slot record fed to the + reference; `accepted` is whether it read; `bits` (bin, 8 bytes, big-endian) is the int64 it read, or the IEEE bits of + the double it read (a float32 widened), and empty when refused. Every value is fed to every leaf. +- `writes.msgpack`: an array of `[leaf, text, bytes]`. `text` is a decimal for an integer leaf and the double's IEEE + bits in hex for a float leaf; `bytes` (bin) is the one-slot record the reference writes for that value. diff --git a/tests/vectors/numeric-leaves/mixed-record.msgpack b/tests/vectors/numeric-leaves/mixed-record.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..8077258bc5d4191f852a5e947f1612444cdd53b4 GIT binary patch literal 20 YcmbQbD9@0cP&?q-&5)LO=m6=aLdjX+RWEAX-F6 zMIb3oy4Gde$Vq^$8TnTR>R@D{fAGkT%Ppo`gn4#a$`NB zOQ1_X1YNU7hjH_vzbNU zHdRe~Q*`!sx%|`eK%VS1 zub#e6qufT4^H%JgPcLb_QhVX^=F!oyLN@IbVRdmfQ!!`xPSh5F?1GE0>2n`)RqA|D#(Bx6=eWM$pB#t-l3K^bN$bu6|vN+9U;{1zD z;9sWEe~^n~SvkUHvSDv6**G+pjYD(U0L^3*ko0^l**Jf>Y#f@)#`)B;ar^kfaFy!L zlx(o4XM=38(z3xakqvJSvf&lW=KcK#kM?5UXf7Ze#!%rTozHa53GT(y?B|5Yr+J(+ zQ9!qT-#tId+_8j4cXA`?C8K%H#BTDX#ic?T7kDkD-6r>~#DvrDwoVV;I2-C_yYsfa z7Ffx@me^>!V87*i5c&U5v0UT7;^G^a)js|J^Ag{cUyz=+Z!-JPtH6&7@wE(zrxD|O zA;MOgR^-qR1nlBPoZw}e*o#cT%QW&{u*5A2d7%f=osr!j$r*V`?67h5Q zkNL;o;`8(J0&NPs`UK4ifV1C93jmfPun^w>lLFAiO93eG?i1DLzn%Hd3&6z!zwq&? z0Hppq3jpW7q5wEvR8KF`f8Xl6or>zg%QW(x1pvMsS1153t||aBjeb`a0DrEu0MNH_ rwC^kc&eq^EY2ZwTC;I(e_u>2>T1bwX literal 0 HcmV?d00001 diff --git a/tests/vectors/numeric-leaves/writes.msgpack b/tests/vectors/numeric-leaves/writes.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..6bb1bad8d84facdbdd5335b40f1a1a9d22ed9c43 GIT binary patch literal 1276 zcmZ{iy>8S%6otJ3A(WR0FJLsD|B*o1md99DYY7PjA(0llq_t9kXdncn&;&t4m%>%p zkifP`Ns$MLsOW}!$1|Ss8m96Ye{;^6YyZmb?j1bZdB*4)(rz?a5Cw>d7yAWOAgd^K zd!t#_K~o`6+o)xGvpJmvvjZsv6S!pqyQ>)am^%G*r~F8(T{omO5>qA z^K~Up1}6DnMouO(lKhZU!;Mtb2n+!KE;@Z9+s)<->|r2%#*raJ6~<5j{fIGzW47>o z@9-)A2eh)6Xp?o{Q1av+20T`HzKR~}Io##kaTXhHoRWVwlrp9$VNydKB{Wfpp$#sN zeaXk1#>m>a{r74ti43hrS3_uJFI=A8`%pf9GJ{|Dh?S)#$$}Z5Y#&2ZLOe9dnjvw2 zxjcGI(pCI;YfYP{>oY$D-G0KHxRI r-C>bZ3#JyF%{7=RVdj|lTwUFJk?p=tsP#DkRb7v~*8dEo{3QMY9GJp# literal 0 HcmV?d00001 From fa1c006beea8c1444e711db3a380c7dc3139dc74 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:34:31 +0200 Subject: [PATCH 18/44] cultcache-ts and cultnet-ts: replication writes a codec document through the codec; numeric oracle read from the C# vectors; float32 rounds once; arm and slot keys span the reference's Int32 - cultnet-ts replication calls cultcache-ts's resolveDocumentFormatter (now exported) and loses its private @msgpack/msgpack default, so a raw put and a snapshot of a codec document carry the codec's bytes. - the hand-written numeric table is deleted; the tests read tests/vectors/numeric-leaves. - an integer past 2^53 in a float32 slot rounds once (ties to even), not through a double. - a union arm key is any Int32 (negative too, sorted numerically); a slot key is 0 to Int32 max. Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/README.md | 6 +- packages/cultcache-ts/src/codec.ts | 52 ++++++--- packages/cultcache-ts/src/index.ts | 2 +- .../cultcache-ts/test/union-parity.test.ts | 90 +++++++++++---- packages/cultnet-ts/src/replication.ts | 18 +-- .../cultnet-ts/test/replication-codec.test.ts | 104 ++++++++++++++++++ 6 files changed, 213 insertions(+), 59 deletions(-) create mode 100644 packages/cultnet-ts/test/replication-codec.test.ts diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 06e1f3f8..1ea366f0 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -272,13 +272,13 @@ const sceneDocument = defineDocumentType({ - The leaves are `string`, `boolean`, `bytes`, `int32`, `int64`, `float32` and `float64`, each writing the MessagePack the C# type writes and refusing what C# refuses (a float in an int slot, an `int` outside Int32). `int64` reads a number when the value is a safe integer and a bigint beyond 2^53, and refuses a number beyond 2^53 on write. `list`, `array` - and `nullable` wrap a leaf, a union or a record. + and `nullable` wrap a leaf, a union or a record. A `float32` slot rounds an integer once, as the reference's cast does. - Every codec carries the reference's type name, and a store's schema catalog is derived from those names, because the reference refuses a store whose slot names another type. The leaves know theirs; give a union or a record its C# name as the last argument of `union(name, arms, typeName)` or `slots(spec, typeName)`. A codec with a slot it cannot name refuses to derive members: pass `members` instead. -- An arm key is an integer within Int32, in any MessagePack integer encoding. A float key (even `1.0`) is refused on - read, and a union refuses a key outside Int32 when it is defined. +- An arm key is an integer within Int32 (negative too), in any MessagePack integer encoding. A float key (even `1.0`) is + refused on read, and a union refuses a key outside Int32 when it is defined. A slot key is within 0 to Int32 max. - `codec` supplies the document's formatter and, unless `members` is given, its catalog members. A definition may not carry both `codec` and `formatter`. `schema.parse` still runs on every decoded value. diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 508fb9d6..c7d2736d 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -87,6 +87,19 @@ export const int32 = integerCodec("System.Int32", "an int32", -(2n ** 31n), 2n * // C# long. A number when the value is a safe integer, a bigint beyond 2^53 (so 64-bit values never lose precision). export const int64 = integerCodec("System.Int64", "an int64", -(2n ** 63n), 2n ** 63n - 1n); +// A 64-bit integer to the nearest float32 in one rounding, ties to even, as the reference's cast does. Number(value) +// first would round to a double and then again, and the two roundings can disagree. +function bigintToFloat32(value: bigint): number { + const magnitude = value < 0n ? -value : value; + const excess = magnitude.toString(2).length - 24; + if (excess <= 0) return Number(value); + const kept = magnitude >> BigInt(excess); + const rest = magnitude - (kept << BigInt(excess)); + const half = 1n << BigInt(excess - 1); + const rounded = Number(rest > half || (rest === half && (kept & 1n) === 1n) ? kept + 1n : kept) * 2 ** excess; + return value < 0n ? -rounded : rounded; +} + function floatCodec(typeName: string, wide: boolean): Codec { const what = wide ? "a float64" : "a float32"; return { @@ -98,7 +111,8 @@ function floatCodec(typeName: string, wide: boolean): Codec { }, decode(raw) { if (raw instanceof WireFloat) return wide ? raw.value : Math.fround(raw.value); - if (typeof raw === "number" || typeof raw === "bigint") return wide ? Number(raw) : Math.fround(Number(raw)); + if (typeof raw === "number") return wide ? raw : Math.fround(raw); + if (typeof raw === "bigint") return wide ? Number(raw) : bigintToFloat32(raw); return refuse(what, kindOf(raw)); }, }; @@ -170,21 +184,25 @@ type SlotSpec = type SlotsSpec = Readonly }[keyof TValue & string]>>; +const INT32_MIN = -(2 ** 31); const INT32_MAX = 2 ** 31 - 1; -// The declared keys, ascending. An arm key is an Int32 in the reference, so a union refuses a wider key at definition. -function slotKeys(spec: Readonly>, owner: string, max = Number.MAX_SAFE_INTEGER): number[] { - // Object.keys lists integer keys in ascending order, so the declared keys come back sorted. - return Object.keys(spec).map((text) => { - const key = Number(text); - if (!Number.isInteger(key) || key < 0 || String(key) !== text) { - throw new Error(`${owner} declares the key "${text}", which is not a non-negative integer.`); - } - if (key > max) { - throw new Error(`${owner} declares the key ${key}, which is outside Int32.`); - } - return key; - }); +// The declared keys, ascending. The reference's keys are Int32: a union arm may be any (a negative one too), a slot is +// non-negative, and a key outside the range is refused at definition rather than allocated for. +function slotKeys(spec: Readonly>, owner: string, low: number): number[] { + return Object.keys(spec) + .map((text) => { + if (!/^-?(0|[1-9][0-9]*)$/u.test(text) || (low >= 0 && text.startsWith("-"))) { + throw new Error(`${owner} declares the key "${text}", which is not ${low < 0 ? "an" : "a non-negative"} integer.`); + } + const key = BigInt(text); + if (key < BigInt(low) || key > BigInt(INT32_MAX)) { + throw new Error(`${owner} declares the key ${text}, which is outside Int32.`); + } + return Number(key); + }) + // Object.keys lists negative keys after the others, in the order they were written: sort them numerically. + .sort((left, right) => left - right); } function pascalCase(name: string): string { @@ -199,7 +217,7 @@ function pascalCase(name: string): string { // are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that // declares a default also takes the default (cultcache-py's rule). export function slots(spec: SlotsSpec, typeName?: string): SlotsCodec { - const keys = slotKeys(spec, "A slot record"); + const keys = slotKeys(spec, "A slot record", 0); if (keys.length === 0) { throw new Error("A slot record declares no slots."); } @@ -271,7 +289,7 @@ export function union( arms: Readonly>>, typeName?: string, ): Codec { - const keys = slotKeys(arms, `Union ${name}`, INT32_MAX); + const keys = slotKeys(arms, `Union ${name}`, INT32_MIN); if (keys.length === 0) { throw new Error(`Union ${name} declares no arms.`); } @@ -314,7 +332,7 @@ export function union( if (typeof key !== "number" && typeof key !== "bigint") { throw new Error(`Union ${name}: the arm key is not an integer (declared keys: ${declared}).`); } - if (key < -(2 ** 31) || key > INT32_MAX) { + if (key < INT32_MIN || key > INT32_MAX) { throw new Error(`Union ${name}: the arm key is outside Int32 (declared keys: ${declared}).`); } const arm = byKey.get(Number(key)); diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 78e9c7c0..75a94f58 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,7 @@ export * as codecs from "./codec"; export type { Codec, SlotsCodec } from "./codec"; export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { defineDocumentRegistry, defineDocumentType } from "./document"; +export { defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; export type { AnyCultCacheDocumentDefinition, diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index cb76ed03..2b4d4dfc 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -12,7 +12,6 @@ import { codecs } from "../src"; import type { Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; -import { referenceReads, referenceWrites } from "./reference-numeric-table"; import { defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; @@ -378,7 +377,7 @@ test("slot names become catalog member names in Pascal case", () => { test("definitions are refused at definition time", () => { assert.throws(() => union<{ kind: "A" }>("U", { 0: ["A", slots<{ kind: string }>({ 0: ["kind", string] })] }), /Union U arm "A" declares a slot named kind/u); assert.throws(() => union<{ kind: "A" | "B" }>("U", { 0: ["A", unit], 1: ["A" as "B", unit] }), /declares the arm "A" twice/u); - assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not a non-negative integer\.$/u); + assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not an integer\.$/u); assert.throws(() => union<{ kind: "A" }>("U", { "01": ["A", unit] } as never), /key "01"/u); assert.throws(() => union<{ kind: "A" }>("U", { "1.5": ["A", unit] } as never), /key "1\.5"/u); assert.throws(() => union<{ kind: "A" }>("U", {} as never), /^Error: Union U declares no arms\.$/u); @@ -532,39 +531,55 @@ test("every leaf and wrapper carries the reference's type name, and a catalog is assert.equal(given.members!.length, 1); }); -// referenceReads and referenceWrites are the C# reference's own answers (see the table's header): a numeric leaf reads -// and refuses exactly what the reference does, and writes exactly the bytes the reference reads back unchanged. +// tests/vectors/numeric-leaves is written by the C# reference (NumericLeafVectorTests): what it reads and refuses for each +// numeric leaf, and the bytes it writes. +const numericVectors = join(vectors, "..", "numeric-leaves"); const numericLeaves: Record> = { int32, int64, float32, float64 }; -const bitsOf = (value: number) => (Number.isNaN(value) ? "NaN" : Buffer.from(new Float64Array([value]).buffer).reverse().toString("hex")); +const doubleBitsHex = (value: number) => { + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value); + return Buffer.from(view.buffer).toString("hex"); +}; +const int64BitsHex = (value: number | bigint) => BigInt.asUintN(64, BigInt(value)).toString(16).padStart(16, "0"); +const isNaNBits = (bitsHex: string) => { + const bits = BigInt(`0x${bitsHex}`); + return ((bits >> 52n) & 0x7ffn) === 0x7ffn && (bits & ((1n << 52n) - 1n)) !== 0n; +}; -test("numeric leaves read and refuse exactly what the C# reference does", () => { - assert.equal(referenceReads.length, 120); +test("numeric leaves read and refuse exactly what the C# reference does", async () => { + const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "reads.msgpack")))) as [string, Uint8Array, boolean, Uint8Array][]; + assert.equal(rows.length, 148); let accepted = 0; let refused = 0; - for (const [leaf, value, outcome] of referenceReads) { + for (const [leaf, payload, wasAccepted, bits] of rows) { + const label = `${leaf} ${hex(payload)}`; const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); - let got: string; - try { - const read = record.decode(readWire(fromHex(`91${value}`))).v; - got = `ACCEPT ${leaf.startsWith("float") ? bitsOf(read as number) : String(read)}`; + if (wasAccepted) { + const read = record.decode(readWire(payload)).v as number | bigint; + const wanted = hex(bits); + if (leaf.startsWith("float")) { + if (isNaNBits(wanted)) assert.ok(Number.isNaN(read), label); + else assert.equal(doubleBitsHex(read as number), wanted, label); + } else { + assert.equal(int64BitsHex(read), wanted, label); + } accepted += 1; - } catch (error) { - assert.ok(!(error as Error).message.includes(value), "a refusal never echoes the value"); - got = "REFUSE"; + } else { + assert.throws(() => record.decode(readWire(payload)), (error: Error) => !error.message.includes(hex(payload).slice(2)), label); refused += 1; } - assert.equal(got, outcome, `${leaf} ${value}`); } - assert.deepEqual([accepted, refused], [referenceReads.filter((row) => row[2].startsWith("ACCEPT")).length, referenceReads.filter((row) => row[2] === "REFUSE").length]); - assert.ok(accepted > 40 && refused > 40); + assert.ok(accepted > 40 && refused > 40, `accepted ${accepted}, refused ${refused}`); + assert.equal(accepted + refused, rows.length); }); -test("numeric leaves write the bytes the C# reference reads back unchanged, and 64-bit values lose nothing", () => { - assert.equal(referenceWrites.length, 46); - for (const [leaf, text, expected] of referenceWrites) { - const value = leaf.startsWith("float") ? (text === "-0" ? -0 : Number(text)) : text.endsWith("n") || Math.abs(Number(text)) > Number.MAX_SAFE_INTEGER ? BigInt(text.replace("n", "")) : Number(text); +test("numeric leaves write the bytes the C# reference writes, and 64-bit values lose nothing", async () => { + const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "writes.msgpack")))) as [string, string, Uint8Array][]; + assert.equal(rows.length, 46); + for (const [leaf, text, expected] of rows) { + const value = leaf.startsWith("float") ? new DataView(Uint8Array.from(Buffer.from(text, "hex")).buffer).getFloat64(0) : leaf === "int32" ? Number(text) : BigInt(text); const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); - assert.equal(hex(writeWire(record.encode({ v: value }))), expected, `${leaf} ${text}`); + assert.equal(hex(writeWire(record.encode({ v: value }))), hex(expected), `${leaf} ${text}`); } const wide = slots<{ v: number | bigint }>({ 0: ["v", int64] }); for (const value of [2n ** 53n + 1n, 2n ** 63n - 1n, -(2n ** 63n), -(2n ** 53n) - 1n]) { @@ -642,3 +657,32 @@ function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entrie async delete() {}, }; } + +test("arm keys span Int32 and are listed ascending; slot keys are non-negative Int32", () => { + const keys = union<{ kind: "High" | "Zero" | "Neg" | "Low" }>("U", { + 2147483647: ["High", unit], + 0: ["Zero", unit], + [-1]: ["Neg", unit], + [-2147483648]: ["Low", unit], + }); + for (const [kind, key] of [["High", 2147483647], ["Zero", 0], ["Neg", -1], ["Low", -2147483648]] as const) { + const written = keys.encode({ kind }); + assert.deepEqual(written, [key, []]); + assert.deepEqual(keys.decode(readWire(writeWire(written))), { kind }); + } + assert.throws(() => keys.decode(readWire(fromHex("92059090"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." }); + assert.throws(() => union<{ kind: "A" }>("U", { [-2147483649]: ["A", unit] }), { message: "Union U declares the key -2147483649, which is outside Int32." }); + assert.throws(() => union<{ kind: "A" }>("U", { 99999999999999999999: ["A", unit] } as never), { message: "Union U declares the key 100000000000000000000, which is outside Int32." }); + assert.doesNotThrow(() => slots<{ a: string }>({ 2147483647: ["a", string] })); + assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is outside Int32." }); + assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), { message: 'A slot record declares the key "-1", which is not a non-negative integer.' }); +}); + +test("the mixed-record vector is the bytes the reference writes for a record of every leaf kind", async () => { + const mixed = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({ + 0: ["id", string], 1: ["i", int32], 2: ["l", int64], 3: ["f", float32], 4: ["d", float64], + }); + const committed = new Uint8Array(await readFile(join(vectors, "..", "numeric-leaves", "mixed-record.msgpack"))); + assert.equal(hex(writeWire(mixed.encode({ id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }))), hex(committed)); + assert.deepEqual(mixed.decode(readWire(committed)), { id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }); +}); diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index a2eda250..fb970906 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -1,12 +1,11 @@ -import { decode, encode } from "@msgpack/msgpack"; import type { AnyCultCacheDocumentDefinition, CultCache, CultCacheDocumentDefinition, - CultCacheDocumentFormatter, CultCacheDocumentValue, CultCacheEnvelope, } from "@gamecult/cultcache-ts"; +import { resolveDocumentFormatter } from "@gamecult/cultcache-ts"; import { type CultNetDocumentDeleteMessage, @@ -136,7 +135,7 @@ export class CultNetDocumentRegistry { } = {}, ): CultNetDocumentPutRawMessage { const parsed = binding.definition.schema.parse(value); - const formatter = formatterFor(binding.definition); + const formatter = resolveDocumentFormatter(binding.definition); return { schemaVersion: "cultnet.document_put_raw.v0", messageId, @@ -435,18 +434,7 @@ function decodeDocumentValue( definition: TDefinition, payload: Uint8Array, ): CultCacheDocumentValue { - const formatter = formatterFor(definition); + const formatter = resolveDocumentFormatter(definition); const decoded = formatter.decode(payload); return definition.schema.parse(decoded) as CultCacheDocumentValue; } - -function formatterFor( - definition: TDefinition, -): CultCacheDocumentFormatter> { - return (definition.formatter as CultCacheDocumentFormatter< - CultCacheDocumentValue - > | undefined) ?? { - encode: (value: CultCacheDocumentValue) => encode(value), - decode: (bytes: Uint8Array) => decode(bytes) as CultCacheDocumentValue, - }; -} diff --git a/packages/cultnet-ts/test/replication-codec.test.ts b/packages/cultnet-ts/test/replication-codec.test.ts new file mode 100644 index 00000000..0c6cd632 --- /dev/null +++ b/packages/cultnet-ts/test/replication-codec.test.ts @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +import { z } from "zod"; +import { CultCache, codecs, defineDocumentType, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; + +import { CultNetDocumentRegistry, defineCultNetDocumentBinding } from "../src"; + +const { float32, float64, int32, int64, slots, string } = codecs; + +// tests/vectors/numeric-leaves/mixed-record.msgpack is the bytes the C# reference writes for Mixed { "n1", 7, 8, 1.5f, 2.5 } +// (NumericLeafVectorTests). A payload equal to it is a payload the reference reads. +function referenceMixedRecord(): Uint8Array { + for (let dir = __dirname; ; dir = resolve(dir, "..")) { + const file = join(dir, "tests", "vectors", "numeric-leaves", "mixed-record.msgpack"); + if (existsSync(file)) return new Uint8Array(readFileSync(file)); + if (resolve(dir, "..") === dir) throw new Error("tests/vectors/numeric-leaves not found above the test."); + } +} + +const mixedCodec = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({ + 0: ["id", string], + 1: ["i", int32], + 2: ["l", int64], + 3: ["f", float32], + 4: ["d", float64], +}); + +const mixedDefinition = defineDocumentType({ + type: "tests.mixed_record", + schemaId: "tests.mixed_record.v1", + schemaName: "tests.mixed_record", + schemaVersion: "tests.mixed_record.v1", + schema: z.object({ id: z.string(), i: z.number(), l: z.union([z.number(), z.bigint()]), f: z.number(), d: z.number() }), + codec: mixedCodec, + name: "id", +}); + +const value = { id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }; +const hex = (data: Uint8Array) => Buffer.from(data).toString("hex"); + +function registry() { + return new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: mixedDefinition })]); +} + +test("a codec document replicates as the bytes the C# reference writes, and a snapshot of it applies on another cache", async () => { + const dir = mkdtempSync(join(tmpdir(), "cultnet-codec-")); + try { + const reference = referenceMixedRecord(); + const documents = registry(); + const origin = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "origin.cc"))).build(); + const target = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "target.cc"))).build(); + + const put = documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-1", "n1", value); + assert.equal(hex(put.document.payload), hex(reference), "a raw put carries the reference's record, not a map"); + + await documents.applyRawDocumentPutMessage(origin, put); + assert.equal(hex(origin.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + + const raw = documents.createRawSnapshotResponse(origin, "snapshot-raw"); + assert.equal(hex(raw.documents[0].payload), hex(reference)); + await documents.applyRawSnapshotResponse(target, raw); + assert.deepEqual(target.getRequired(mixedDefinition, "n1"), value); + assert.equal(hex(target.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + + // The value-level snapshot decodes the stored record through the codec, so a peer applying it parses a record. + const values = documents.createSnapshotResponse(origin, "snapshot-values"); + assert.deepEqual(values.documents[0].payload, value); + const other = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "other.cc"))).build(); + await documents.applySnapshotResponse(other, values); + assert.equal(hex(other.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a definition that carries a codec and a formatter is refused where replication reads it", () => { + const both = { + ...mixedDefinition, + formatter: { encode: () => new Uint8Array(), decode: () => value }, + } as unknown as typeof mixedDefinition; + const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: both })]); + assert.throws( + () => documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-2", "n1", value), + { message: 'CultCache document type "tests.mixed_record" declares both a codec and a formatter.' }, + ); +}); + +test("a document with neither a codec nor a formatter replicates as a map, as before", () => { + const plain = defineDocumentType({ + type: "tests.plain_record", + schemaId: "tests.plain_record.v1", + schemaName: "tests.plain_record", + schemaVersion: "tests.plain_record.v1", + schema: z.object({ id: z.string() }), + name: "id", + }); + const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: plain })]); + const put = documents.createRawDocumentPutMessage(documents.get("tests.plain_record")!, "put-3", "n1", { id: "n1" }); + assert.equal(hex(put.document.payload), "81a26964a26e31"); +}); From 44dbf413627ab18ffc0d24ba733f1731b655f3e6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:39:42 +0200 Subject: [PATCH 19/44] union-parity test: the unknown-arm payload is one array, not two values Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/test/union-parity.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 2b4d4dfc..69727b8f 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -670,7 +670,7 @@ test("arm keys span Int32 and are listed ascending; slot keys are non-negative I assert.deepEqual(written, [key, []]); assert.deepEqual(keys.decode(readWire(writeWire(written))), { kind }); } - assert.throws(() => keys.decode(readWire(fromHex("92059090"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." }); + assert.throws(() => keys.decode(readWire(fromHex("920590"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." }); assert.throws(() => union<{ kind: "A" }>("U", { [-2147483649]: ["A", unit] }), { message: "Union U declares the key -2147483649, which is outside Int32." }); assert.throws(() => union<{ kind: "A" }>("U", { 99999999999999999999: ["A", unit] } as never), { message: "Union U declares the key 100000000000000000000, which is outside Int32." }); assert.doesNotThrow(() => slots<{ a: string }>({ 2147483647: ["a", string] })); From 6640b9d358740eae541247635fc72e0edd21a119 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:45:04 +0200 Subject: [PATCH 20/44] NumericLeafVectorTests: an odd float32 value just past exact, the case a wrong tie rule rounds up Co-Authored-By: Claude Opus 5.5 --- tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index 290d433e..912fdb56 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -72,8 +72,8 @@ namespace GameCult.Caching.Tests { "05", "ff", "ccc8", "cd0100", "ceb2d05e00", "d0ce", "d18000", "d280000000", "d38000000000000000", "cf0020000000000001", "cfffffffffffffffff", "cf8000000000000000", "d30020000000000001", "d3fffffffffffffff0", - // integers beyond 2^53 that a float32 slot must round once: past a tie, on a tie to even, on a tie to odd, negative - "d31000001000000001", "cf8000008000000001", "cf8000008000000000", "cf8000018000000000", "d3efffffe800000001", "d3effffff000000001", + // integers beyond 2^53 that a float32 slot must round once: past a tie, on a tie to even, on a tie to odd, an odd value just past exact, negative + "d31000001000000001", "cf8000008000000001", "cf8000008000000000", "cf8000018000000000", "cf8000010000000001", "cf8000010000000000", "d3efffffe800000001", "d3effffff000000001", "ca3fc00000", "cb3ff8000000000000", "cb401c000000000000", "cb7ff8000000000000", "ca7f800000", "ca7f800001", "cb7e37e43c8800759c", "cb47efffffe0000000", "cb3fb999999999999a", "ca3dcccccd", "cb8000000000000000", "c0", "c3", "a161", "c40100", "90", "80", @@ -209,6 +209,8 @@ namespace GameCult.Caching.Tests [TestCase("float32", "cf8000008000000001", "43e0000020000000")] [TestCase("float32", "cf8000008000000000", "43e0000000000000")] [TestCase("float32", "cf8000018000000000", "43e0000040000000")] + [TestCase("float32", "cf8000010000000001", "43e0000020000000")] + [TestCase("float32", "cf8000010000000000", "43e0000020000000")] [TestCase("float64", "cf0020000000000001", "4340000000000000")] [TestCase("int64", "cf0020000000000001", "0020000000000001")] public void ReadsRoundExactlyAsTheReferenceDoes(string leaf, string value, string bitsHex) From 26ff9d27e98e08ecd3f3258362b0b6016e1c733a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 08:45:48 +0200 Subject: [PATCH 21/44] numeric-leaves vectors gain two float32 rows; float32 integer rounding drops its redundant guard An odd kept value just past exact pins the tie rule (the Stryker survivor). The excess <= 0 guard was equivalent to the general path for the only values that reach it, so it is deleted. Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 4 ++-- .../cultcache-ts/test/union-parity.test.ts | 2 +- tests/vectors/numeric-leaves/reads.msgpack | Bin 3643 -> 3859 bytes 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index c7d2736d..1ad50f8e 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -88,11 +88,11 @@ export const int32 = integerCodec("System.Int32", "an int32", -(2n ** 31n), 2n * export const int64 = integerCodec("System.Int64", "an int64", -(2n ** 63n), 2n ** 63n - 1n); // A 64-bit integer to the nearest float32 in one rounding, ties to even, as the reference's cast does. Number(value) -// first would round to a double and then again, and the two roundings can disagree. +// first would round to a double and then again, and the two roundings can disagree. A bigint reaches here only beyond +// 2^53 (the wire reader returns a number for any smaller integer), so at least 30 bits are rounded away. function bigintToFloat32(value: bigint): number { const magnitude = value < 0n ? -value : value; const excess = magnitude.toString(2).length - 24; - if (excess <= 0) return Number(value); const kept = magnitude >> BigInt(excess); const rest = magnitude - (kept << BigInt(excess)); const half = 1n << BigInt(excess - 1); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 69727b8f..6b31c902 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -548,7 +548,7 @@ const isNaNBits = (bitsHex: string) => { test("numeric leaves read and refuse exactly what the C# reference does", async () => { const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "reads.msgpack")))) as [string, Uint8Array, boolean, Uint8Array][]; - assert.equal(rows.length, 148); + assert.equal(rows.length, 156); let accepted = 0; let refused = 0; for (const [leaf, payload, wasAccepted, bits] of rows) { diff --git a/tests/vectors/numeric-leaves/reads.msgpack b/tests/vectors/numeric-leaves/reads.msgpack index 45aaf7f42e772f4b881bdbe5a85491143d7996b1..51991ff24fb9298d58855b84582c0119ffd9cc1e 100644 GIT binary patch delta 162 zcmdljGg*%L4#S*{%u5)h&o?kIG5`VNp(6}amS*Oa7#khong|x%JdM$UnH9{OJds6& ziIHLRUY2Rhd~hv?k8n6YnEa4klpAaW1B9Q*AvQUJhi7v#$46#ERJ9BY3P7u-EKkeH WPb@JrfmsByAFLNBzuA@N8zTTYh%X)h delta 32 ocmbO%w_Aq!4#Sj<%u5(IUuINi-mJ#j%DnkC2P4a79p2}R0KKpZ>i_@% From 1e0c5a920977a0bdd6b4fc27facb757eae7cd022 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 09:18:18 +0200 Subject: [PATCH 22/44] batch 4 code: mesh readers decode through the definition's formatter, float32 sNaN refused, slot keys capped at 65535 cultmesh-ts reads document payloads through resolveDocumentFormatter at its five plain-msgpack sites; WireFloat carries a signalling flag and the float32 leaf refuses it; slots() refuses a key above 65535 at definition. The C# vector writer gains the 2^24+1 reads row and float32-nan.msgpack, whose committed files land in the next commit: this commit does not build its tests (union-parity numeric vectors, row count 160 and float32-nan.msgpack are absent). Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 22 +++-- packages/cultcache-ts/src/wire.ts | 9 +- .../cultcache-ts/test/union-parity.test.ts | 54 +++++++++- packages/cultmesh-ts/src/index.ts | 27 +++-- .../cultmesh-ts/test/codec-documents.test.ts | 98 +++++++++++++++++++ .../NumericLeafVectorTests.cs | 37 +++++++ 6 files changed, 226 insertions(+), 21 deletions(-) create mode 100644 packages/cultmesh-ts/test/codec-documents.test.ts diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 1ad50f8e..7dba8edc 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -110,7 +110,11 @@ function floatCodec(typeName: string, wide: boolean): Codec { return new WireFloat(value, wide); }, decode(raw) { - if (raw instanceof WireFloat) return wide ? raw.value : Math.fround(raw.value); + if (raw instanceof WireFloat) { + // A JS number has no NaN payload, so a signalling float32 NaN cannot be kept; refusing beats changing its bytes. + if (raw.signalling && !wide) refuse(what, "a signalling NaN"); + return wide ? raw.value : Math.fround(raw.value); + } if (typeof raw === "number") return wide ? raw : Math.fround(raw); if (typeof raw === "bigint") return wide ? Number(raw) : bigintToFloat32(raw); return refuse(what, kindOf(raw)); @@ -187,17 +191,21 @@ type SlotsSpec = Readonly>, owner: string, low: number): number[] { +// 0 to MAX_SLOT_KEY, and a key outside its range is refused at definition rather than allocated for. +function slotKeys(spec: Readonly>, owner: string, low: number, high: number): number[] { return Object.keys(spec) .map((text) => { if (!/^-?(0|[1-9][0-9]*)$/u.test(text) || (low >= 0 && text.startsWith("-"))) { throw new Error(`${owner} declares the key "${text}", which is not ${low < 0 ? "an" : "a non-negative"} integer.`); } const key = BigInt(text); - if (key < BigInt(low) || key > BigInt(INT32_MAX)) { - throw new Error(`${owner} declares the key ${text}, which is outside Int32.`); + if (key < BigInt(low) || key > BigInt(high)) { + throw new Error(`${owner} declares the key ${text}, which is ${high === INT32_MAX ? "outside Int32" : `above the slot key limit ${high}`}.`); } return Number(key); }) @@ -217,7 +225,7 @@ function pascalCase(name: string): string { // are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that // declares a default also takes the default (cultcache-py's rule). export function slots(spec: SlotsSpec, typeName?: string): SlotsCodec { - const keys = slotKeys(spec, "A slot record", 0); + const keys = slotKeys(spec, "A slot record", 0, MAX_SLOT_KEY); if (keys.length === 0) { throw new Error("A slot record declares no slots."); } @@ -289,7 +297,7 @@ export function union( arms: Readonly>>, typeName?: string, ): Codec { - const keys = slotKeys(arms, `Union ${name}`, INT32_MIN); + const keys = slotKeys(arms, `Union ${name}`, INT32_MIN, INT32_MAX); if (keys.length === 0) { throw new Error(`Union ${name} declares no arms.`); } diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index d9102237..12bf572a 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -4,7 +4,8 @@ // outside the safe range. A float decodes to a WireFloat, whatever its value. export class WireFloat { - constructor(readonly value: number, readonly wide: boolean) {} + // signalling: a float32 NaN whose quiet bit is clear. The JS number it becomes has lost that payload. + constructor(readonly value: number, readonly wide: boolean, readonly signalling = false) {} } const SAFE_MIN = BigInt(Number.MIN_SAFE_INTEGER); @@ -104,7 +105,11 @@ export function readWire(payload: Uint8Array): unknown { case 0xc4: return binary(view.getUint8(take(1))); case 0xc5: return binary(view.getUint16(take(2))); case 0xc6: return binary(view.getUint32(take(4))); - case 0xca: return new WireFloat(view.getFloat32(take(4)), false); + case 0xca: { + const at = take(4); + const bits = view.getUint32(at); + return new WireFloat(view.getFloat32(at), false, (bits & 0x7f800000) === 0x7f800000 && (bits & 0x007fffff) !== 0 && (bits & 0x00400000) === 0); + } case 0xcb: return new WireFloat(view.getFloat64(take(8)), true); case 0xcc: return view.getUint8(take(1)); case 0xcd: return view.getUint16(take(2)); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 6b31c902..c0f9524e 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -9,7 +9,7 @@ import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; import { codecs } from "../src"; -import type { Codec } from "../src/codec"; +import { MAX_SLOT_KEY, type Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; import { defineDocumentType } from "../src/document"; @@ -546,15 +546,27 @@ const isNaNBits = (bitsHex: string) => { return ((bits >> 52n) & 0x7ffn) === 0x7ffn && (bits & ((1n << 52n) - 1n)) !== 0n; }; +const isSignallingFloat32 = (payload: Uint8Array) => { + // payload is a one-slot record [float32]: 91 ca b0 b1 b2 b3 + if (payload.length !== 6 || payload[1] !== 0xca) return false; + const bits = new DataView(payload.buffer, payload.byteOffset).getUint32(2); + return (bits & 0x7f800000) === 0x7f800000 && (bits & 0x007fffff) !== 0 && (bits & 0x00400000) === 0; +}; + test("numeric leaves read and refuse exactly what the C# reference does", async () => { const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "reads.msgpack")))) as [string, Uint8Array, boolean, Uint8Array][]; - assert.equal(rows.length, 156); + assert.equal(rows.length, 160); let accepted = 0; let refused = 0; for (const [leaf, payload, wasAccepted, bits] of rows) { const label = `${leaf} ${hex(payload)}`; const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); - if (wasAccepted) { + if (wasAccepted && leaf === "float32" && isSignallingFloat32(payload)) { + // The one reading that differs from the reference: it reads a float32 signalling NaN, this runtime refuses it + // (float32-nan.msgpack shows the reference keeps its bits, which a JS number cannot). + assert.throws(() => record.decode(readWire(payload)), (error: Error) => !error.message.includes(hex(payload).slice(2)), label); + refused += 1; + } else if (wasAccepted) { const read = record.decode(readWire(payload)).v as number | bigint; const wanted = hex(bits); if (leaf.startsWith("float")) { @@ -573,6 +585,25 @@ test("numeric leaves read and refuse exactly what the C# reference does", async assert.equal(accepted + refused, rows.length); }); +// float32-nan.msgpack: [payload, bytes the reference writes after reading it] for a float32 slot holding an infinity or a NaN. +// The reference keeps every payload bit; a JS number cannot keep a signalling NaN's, so that read is refused. +test("a float32 NaN round-trips with its bits, and a signalling one is refused on read", async () => { + const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "float32-nan.msgpack")))) as [Uint8Array, Uint8Array][]; + assert.equal(rows.length, 6); + const record = slots<{ v: number }>({ 0: ["v", float32] }); + let refused = 0; + for (const [payload, rewritten] of rows) { + if (isSignallingFloat32(payload)) { + assert.equal(hex(rewritten), hex(payload), "the reference keeps the signalling bits"); + assert.throws(() => record.decode(readWire(payload)), { message: "v (slot 0): expected a float32, found a signalling NaN" }, hex(payload)); + refused += 1; + } else { + assert.equal(hex(writeWire(record.encode(record.decode(readWire(payload))))), hex(rewritten), hex(payload)); + } + } + assert.equal(refused, 3); +}); + test("numeric leaves write the bytes the C# reference writes, and 64-bit values lose nothing", async () => { const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "writes.msgpack")))) as [string, string, Uint8Array][]; assert.equal(rows.length, 46); @@ -673,11 +704,24 @@ test("arm keys span Int32 and are listed ascending; slot keys are non-negative I assert.throws(() => keys.decode(readWire(fromHex("920590"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." }); assert.throws(() => union<{ kind: "A" }>("U", { [-2147483649]: ["A", unit] }), { message: "Union U declares the key -2147483649, which is outside Int32." }); assert.throws(() => union<{ kind: "A" }>("U", { 99999999999999999999: ["A", unit] } as never), { message: "Union U declares the key 100000000000000000000, which is outside Int32." }); - assert.doesNotThrow(() => slots<{ a: string }>({ 2147483647: ["a", string] })); - assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is outside Int32." }); + assert.throws(() => slots<{ a: string }>({ 2147483647: ["a", string] }), { message: "A slot record declares the key 2147483647, which is above the slot key limit 65535." }); + assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is above the slot key limit 65535." }); assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), { message: 'A slot record declares the key "-1", which is not a non-negative integer.' }); }); +// A slot record is a dense array of key+1 slots, as the reference writes it, so the key is capped: the largest accepted +// key writes a 65536-slot payload, one more is refused at definition. +test("a slot key is capped at 65535: the cap is the largest slot record, one past it is refused at definition", () => { + assert.equal(MAX_SLOT_KEY, 65535); + const edge = slots<{ a: string }>({ 65535: ["a", string] }); + const written = edge.encode({ a: "x" }) as unknown[]; + assert.equal(written.length, 65536); + assert.equal(written[65535], "x"); + assert.equal(written.filter((slot) => slot === null).length, 65535); + assert.deepEqual(edge.decode(readWire(writeWire(written))), { a: "x" }); + assert.throws(() => slots<{ a: string }>({ 65536: ["a", string] }), { message: "A slot record declares the key 65536, which is above the slot key limit 65535." }); +}); + test("the mixed-record vector is the bytes the reference writes for a record of every leaf kind", async () => { const mixed = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({ 0: ["id", string], 1: ["i", int32], 2: ["l", int64], 3: ["f", float32], 4: ["d", float64], diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 36a5ae88..9c77e586 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -8,6 +8,7 @@ import { type CacheBackingStore, type CultCacheEnvelope, type CultCacheDocumentValue, + resolveDocumentFormatter, } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, @@ -2637,7 +2638,7 @@ export function cultMeshDocumentFromStore( if (!record) { throw new Error(`CultMesh store document '${documentId}' did not contain schema ${resolvedSchemaId}.`); } - return parseCultMeshDocumentValue(schemaOrDefinition, decode(record.payload)); + return parseCultMeshDocumentValue(schemaOrDefinition, decodeCultMeshPayload(schemaOrDefinition, record.payload)); }; return cultMeshDocument( @@ -2750,6 +2751,7 @@ export function cultMeshDocumentFromPeerSnapshot( parseCultMeshDocumentValue( schemaOrDefinition, decodeCultNetRawDocumentPayload( + schemaOrDefinition, await requestCultNetRawSnapshotDocument( peer, resolvedSchemaId, @@ -2761,7 +2763,7 @@ export function cultMeshDocumentFromPeerSnapshot( ? undefined : record => { try { - schemaOrDefinition.schema.parse(decodeCultNetRawDocumentPayload(record)); + schemaOrDefinition.schema.parse(decodeCultNetRawDocumentPayload(schemaOrDefinition, record)); return true; } catch { return false; @@ -4106,7 +4108,7 @@ export class CultMeshNode { messageIdPrefix: options.messageIdPrefix ?? key, accepts: record => { try { - definition.schema.parse(decodeCultNetRawDocumentPayload(record)); + definition.schema.parse(decodeCultNetRawDocumentPayload(definition, record)); return true; } catch { return false; @@ -6329,7 +6331,7 @@ export class CultMesh { const onMessage = (message: CultNetMessage): void => { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || message.messageId !== messageId || message.document.schemaId !== receiptBinding.definition.schemaId) return; try { - const decoded = decode(new Uint8Array(message.document.payload)); + const decoded = decodeCultMeshPayload(receiptBinding.definition, new Uint8Array(message.document.payload)); cleanup(); resolve(receiptBinding.definition.schema.parse(decoded) as CultCacheDocumentValue); } catch (error) { cleanup(); reject(error); } @@ -6638,7 +6640,7 @@ function cultMeshEnvelopeParsesAsDefinition( definition: AnyCultCacheDocumentDefinition, ): boolean { try { - definition.schema.parse(decode(envelope.payload)); + definition.schema.parse(decodeCultMeshPayload(definition, envelope.payload)); return true; } catch { return false; @@ -6774,11 +6776,22 @@ async function resolveCultNetPeer( : peerOrProvider; } -function decodeCultNetRawDocumentPayload(document: CultNetRawDocumentRecord): unknown { +// A document payload is read the way its definition's cache reads it (a codec or formatter document is not a plain +// msgpack map); a payload with no definition is plain msgpack. +function decodeCultMeshPayload(schemaOrDefinition: string | AnyCultCacheDocumentDefinition, payload: Uint8Array): unknown { + return typeof schemaOrDefinition === "string" + ? decode(payload) + : resolveDocumentFormatter(schemaOrDefinition).decode(payload); +} + +function decodeCultNetRawDocumentPayload( + schemaOrDefinition: string | AnyCultCacheDocumentDefinition, + document: CultNetRawDocumentRecord, +): unknown { if (document.payloadEncoding !== "messagepack") { throw new Error(`Unsupported CultNet raw document payload encoding ${document.payloadEncoding}.`); } - return decode(toUint8Array(document.payload)); + return decodeCultMeshPayload(schemaOrDefinition, toUint8Array(document.payload)); } function parseCultMeshDocumentValue( diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts new file mode 100644 index 00000000..0514cb99 --- /dev/null +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { z } from "zod"; +import { codecs, defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + defineCultNetDocumentBinding, + type CultNetPeer, +} from "cultnet-ts"; +import { CultMesh, cultMeshDocumentFromSingleFile } from "../src/index"; + +// A document written in the reference's positional shape, and a plain map document beside it. Every mesh reader of a +// payload reads it the way the cache does, so both read through the mesh. +const codecNote = defineDocumentType({ + type: "cultmesh.codec_note", + schemaId: "cultmesh.codec_note.v0", + schema: z.object({ noteId: z.string(), body: z.string() }), + name: "noteId", + codec: codecs.slots<{ noteId: string; body: string }>({ + 0: ["noteId", codecs.string], + 1: ["body", codecs.string], + }, "Tests.CodecNote"), +}); + +const codecReceipt = defineDocumentType({ + type: "cultmesh.codec_receipt", + schemaId: "cultmesh.codec_receipt.v0", + schema: z.object({ commandId: z.string(), body: z.string() }), + name: "commandId", + codec: codecs.slots<{ commandId: string; body: string }>({ + 0: ["commandId", codecs.string], + 1: ["body", codecs.string], + }, "Tests.CodecReceipt"), +}); + +const plainNote = defineDocumentType({ + type: "cultmesh.plain_note", + schemaId: "cultmesh.plain_note.v0", + schema: z.object({ noteId: z.string(), body: z.string() }), + name: "noteId", +}); + +test("the mesh reads a stored codec document as the cache does, and a plain document unchanged", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-store-")); + const codecPath = join(dir, "codec.ccmp"); + const plainPath = join(dir, "plain.ccmp"); + const codecNode = await CultMesh.startNode(codecPath, { documents: [codecNote] }); + await codecNode.put(codecNote, "n1", { noteId: "n1", body: "positional" }); + const plainNode = await CultMesh.startNode(plainPath, { documents: [plainNote] }); + await plainNode.put(plainNote, "n2", { noteId: "n2", body: "map" }); + + assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" }); + assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" }); +}); + +test("the mesh syncs a codec document from a peer snapshot and returns a codec receipt", async () => { + const connectionId = 0x10203060; + const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-sync-")); + const source = await CultMesh.startNode(join(dir, "source.ccmp"), { documents: [codecNote] }); + await source.put(codecNote, "n1", { noteId: "n1", body: "synced positional" }); + const target = await CultMesh.startNode(join(dir, "target.ccmp"), { documents: [codecNote] }); + const commandBinding = defineCultNetDocumentBinding({ definition: codecNote }); + const receiptBinding = defineCultNetDocumentBinding({ definition: codecReceipt }); + const server = CultMesh.createRudpDocumentServer("cultmesh-ts-codec-server", connectionId, { + documents: new CultNetDocumentRegistry([commandBinding, receiptBinding]), + getCache: () => source.cache, + bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, + onDocumentPutRaw: async (document) => ({ + binding: receiptBinding, + recordKey: document.recordKey, + value: { commandId: document.recordKey, body: "applied" }, + }), + }); + let peer: CultNetPeer | undefined; + try { + await server.start(); + peer = await CultMesh.createRudpPeer("cultmesh-ts-codec-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000, + }); + // The accepts probe reads the snapshot's payload as the codec: a plain decode would reject every codec snapshot. + const synced = await target.syncDocumentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }); + assert.deepEqual(synced, { noteId: "n1", body: "synced positional" }); + assert.deepEqual(target.getRequired(codecNote, "n1"), { noteId: "n1", body: "synced positional" }); + + const receipt = await CultMesh.publishRudpDocumentAndWaitForReceipt( + "cultmesh-ts-codec-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`, + commandBinding, "n2", { noteId: "n2", body: "command" }, receiptBinding, + { messageId: "n2", receiptTimeoutMs: 1_000, resendPollMs: 5 }, + ); + assert.deepEqual(receipt, { commandId: "n2", body: "applied" }); + } finally { + peer?.close(); + server.close(); + } +}); diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index 912fdb56..75b078d7 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -74,6 +74,8 @@ namespace GameCult.Caching.Tests "cf0020000000000001", "cfffffffffffffffff", "cf8000000000000000", "d30020000000000001", "d3fffffffffffffff0", // integers beyond 2^53 that a float32 slot must round once: past a tie, on a tie to even, on a tie to odd, an odd value just past exact, negative "d31000001000000001", "cf8000008000000001", "cf8000008000000000", "cf8000018000000000", "cf8000010000000001", "cf8000010000000000", "d3efffffe800000001", "d3effffff000000001", + // an integer a float32 cannot hold that a reader holds as a plain number: 2^24 + 1 rounds to 2^24 + "ce01000001", "ca3fc00000", "cb3ff8000000000000", "cb401c000000000000", "cb7ff8000000000000", "ca7f800000", "ca7f800001", "cb7e37e43c8800759c", "cb47efffffe0000000", "cb3fb999999999999a", "ca3dcccccd", "cb8000000000000000", "c0", "c3", "a161", "c40100", "90", "80", @@ -174,11 +176,46 @@ namespace GameCult.Caching.Tests // mixed-record.msgpack: the reference's bytes for Mixed { "n1", 7, 8, 1.5f, 2.5 }, which other runtimes must write identically. private static byte[] MixedRecord() => MessagePackSerializer.Serialize(new Mixed { Id = "n1", I = 7, L = 8, F = 1.5f, D = 2.5 }, Options); + // float32-nan.msgpack: an array of [payload (bin), bytes (bin)]: a one-slot float32 record holding an infinity or a NaN, + // and the bytes the reference writes after reading it. A signalling NaN (quiet bit clear) comes back unchanged. + private static readonly string[] Float32NanValues = { "ca7f800000", "ca7f800001", "ca7fa00000", "ca7fc00001", "ca7fe00000", "caff800001" }; + + private static byte[] Float32NanVector() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(Float32NanValues.Length); + foreach (var value in Float32NanValues) + { + var payload = Payload(value); + writer.WriteArrayHeader(2); + writer.Write(payload); + writer.Write(MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(payload, Options), Options)); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + // ---- tests ---- [Test] public void ReadsVectorMatches() => Check("reads.msgpack", ReadsVector()); + [Test] + public void Float32NanVectorMatches() => Check("float32-nan.msgpack", Float32NanVector()); + + [Test] + public void Float32SignallingNanIsReadAndWrittenBitForBit() + { + foreach (var value in new[] { "ca7f800001", "ca7fa00000", "caff800001" }) + { + var payload = Payload(value); + var again = MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(payload, Options), Options); + Assert.That(Convert.ToHexString(again), Is.EqualTo(Convert.ToHexString(payload)), value); + } + } + [Test] public void WritesVectorMatches() => Check("writes.msgpack", WritesVector()); From 5d2a880ffc83362e065ea5648fddad984cc197d8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 09:31:45 +0200 Subject: [PATCH 23/44] numeric-leaves vectors: the 2^24+1 reads row and float32-nan.msgpack, written by the C# reference Regenerated with CULTCACHE_WRITE_VECTORS=1 on Yggdrasil (dotnet 10 image, 16 vector tests green). Completes the previous commit; both build, and the TS suites are checked once at the end. Co-Authored-By: Claude Opus 5.5 --- tests/vectors/numeric-leaves/README.md | 2 ++ .../numeric-leaves/float32-nan.msgpack | Bin 0 -> 103 bytes tests/vectors/numeric-leaves/reads.msgpack | Bin 3859 -> 3967 bytes 3 files changed, 2 insertions(+) create mode 100644 tests/vectors/numeric-leaves/float32-nan.msgpack diff --git a/tests/vectors/numeric-leaves/README.md b/tests/vectors/numeric-leaves/README.md index e7ecb070..3007204e 100644 --- a/tests/vectors/numeric-leaves/README.md +++ b/tests/vectors/numeric-leaves/README.md @@ -12,3 +12,5 @@ A document with one slot, `[value]`, whose slot is a C# `int`, `long`, `float` o the double it read (a float32 widened), and empty when refused. Every value is fed to every leaf. - `writes.msgpack`: an array of `[leaf, text, bytes]`. `text` is a decimal for an integer leaf and the double's IEEE bits in hex for a float leaf; `bytes` (bin) is the one-slot record the reference writes for that value. +- `float32-nan.msgpack`: an array of `[payload, bytes]`. `payload` (bin) is a one-slot `float` record holding an infinity + or a NaN; `bytes` (bin) is what the reference writes after reading it. It keeps every bit, a signalling NaN's included. diff --git a/tests/vectors/numeric-leaves/float32-nan.msgpack b/tests/vectors/numeric-leaves/float32-nan.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..6c65acba4ca5f97deb9b3c750c6a726bb63a4f13 GIT binary patch literal 103 zcmbO>=?L4zQ}qoD3@{odzz7v!ga|BvssPdufddQ>6+jvy@Bpd;NP`9bH$YT?XaMY; BEI9xG literal 0 HcmV?d00001 diff --git a/tests/vectors/numeric-leaves/reads.msgpack b/tests/vectors/numeric-leaves/reads.msgpack index 51991ff24fb9298d58855b84582c0119ffd9cc1e..36f7f061cbdf106e3cc31021471cba322e6f77fb 100644 GIT binary patch delta 89 zcmbO%_g{|r4#R?t%zqdK&M`7DFdja_!2kpx=H{=Axy-yMqMJimuQ7AMgd7VdZ{!r- Qti$QRf+V&%igza?08vF4Z~y=R delta 32 ocmew_H(8GP4#S*{%zqd+OEM)hZ=S|_l6iAArz6Ye8@$UI0nVxn1poj5 From cf9d2fd2b58b0f6193e665691b5e3e939d3df791 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 09:38:56 +0200 Subject: [PATCH 24/44] codec-documents test: the alias record is found by parsing, and a peer snapshot handle reads a codec document Builds; TS suites run at the end. Co-Authored-By: Claude Opus 5.5 --- packages/cultmesh-ts/test/codec-documents.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index 0514cb99..438e7f0e 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -36,6 +36,15 @@ const codecReceipt = defineDocumentType({ }, "Tests.CodecReceipt"), }); +// The same record under another schema id: the store holds it as codec_note, so it is found by parsing, not by id. +const codecNoteAlias = defineDocumentType({ + type: "cultmesh.codec_note_alias", + schemaId: "cultmesh.codec_note_alias.v0", + schema: z.object({ noteId: z.string(), body: z.string() }), + name: "noteId", + codec: codecNote.codec, +}); + const plainNote = defineDocumentType({ type: "cultmesh.plain_note", schemaId: "cultmesh.plain_note.v0", @@ -53,6 +62,7 @@ test("the mesh reads a stored codec document as the cache does, and a plain docu await plainNode.put(plainNote, "n2", { noteId: "n2", body: "map" }); assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" }); + assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" }); assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" }); }); @@ -84,6 +94,8 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r const synced = await target.syncDocumentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }); assert.deepEqual(synced, { noteId: "n1", body: "synced positional" }); assert.deepEqual(target.getRequired(codecNote, "n1"), { noteId: "n1", body: "synced positional" }); + // A peer snapshot handle reads the same way, and accepts the snapshot by parsing it as the codec. + assert.deepEqual(await CultMesh.documentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }).latest(), { noteId: "n1", body: "synced positional" }); const receipt = await CultMesh.publishRudpDocumentAndWaitForReceipt( "cultmesh-ts-codec-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`, From 8d5c3c092349c473d2b54ea1fe7edf12be025456 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 09:47:01 +0200 Subject: [PATCH 25/44] union-parity test: only a quiet-bit-clear NaN is signalling; a finite float32 with bit 22 clear still reads Builds; kills the Stryker survivor at wire.ts:111 (the exponent test replaced by true). Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/test/union-parity.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index c0f9524e..5fa663b3 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -604,6 +604,16 @@ test("a float32 NaN round-trips with its bits, and a signalling one is refused o assert.equal(refused, 3); }); +test("only a NaN with its quiet bit clear is signalling: a finite float32 with a clear bit 22 reads, and a float64 slot reads any float32 NaN", () => { + const f32 = slots<{ v: number }>({ 0: ["v", float32] }); + const f64 = slots<{ v: number }>({ 0: ["v", float64] }); + assert.equal(f32.decode(readWire(fromHex("91ca3f800001"))).v, Math.fround(1.0000001192092896)); + assert.equal(f32.decode(readWire(fromHex("91ca00000001"))).v, Math.fround(1e-45)); + assert.equal(f32.decode(readWire(fromHex("91ca7f800000"))).v, Infinity); + assert.ok(Number.isNaN(f32.decode(readWire(fromHex("91ca7fc00001"))).v)); + assert.ok(Number.isNaN(f64.decode(readWire(fromHex("91ca7f800001"))).v)); +}); + test("numeric leaves write the bytes the C# reference writes, and 64-bit values lose nothing", async () => { const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "writes.msgpack")))) as [string, string, Uint8Array][]; assert.equal(rows.length, 46); From 8fa2d110428ce48c2eb464d9bbc3cf2258a0cbae Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 09:54:10 +0200 Subject: [PATCH 26/44] codec-documents test: the alias found by parsing skips a plain record, and the snapshot probe picks the codec record among two Builds; run at the end. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-ts/test/codec-documents.test.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index 438e7f0e..4a0d2494 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -63,19 +63,30 @@ test("the mesh reads a stored codec document as the cache does, and a plain docu assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" }); assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" }); + // A store holding a plain record first and a codec record second: the alias, found by parsing, takes the codec one. + const bothPath = join(dir, "both.ccmp"); + const bothNode = await CultMesh.startNode(bothPath, { documents: [plainNote, codecNote] }); + await bothNode.put(plainNote, "n2", { noteId: "n2", body: "map" }); + await bothNode.put(codecNote, "n1", { noteId: "n1", body: "positional" }); + assert.deepEqual(await cultMeshDocumentFromSingleFile(bothPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" }); assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" }); }); test("the mesh syncs a codec document from a peer snapshot and returns a codec receipt", async () => { const connectionId = 0x10203060; const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-sync-")); - const source = await CultMesh.startNode(join(dir, "source.ccmp"), { documents: [codecNote] }); + const source = await CultMesh.startNode(join(dir, "source.ccmp"), { documents: [codecNote, plainNote] }); await source.put(codecNote, "n1", { noteId: "n1", body: "synced positional" }); + // Two records at one key, in two shapes: a peer snapshot handle for the alias schema (which neither holds) accepts the + // one that parses as the codec. + await source.put(plainNote, "n3", { noteId: "n3", body: "map" }); + await source.put(codecNote, "n3", { noteId: "n3", body: "positional" }); const target = await CultMesh.startNode(join(dir, "target.ccmp"), { documents: [codecNote] }); const commandBinding = defineCultNetDocumentBinding({ definition: codecNote }); + const plainBinding = defineCultNetDocumentBinding({ definition: plainNote }); const receiptBinding = defineCultNetDocumentBinding({ definition: codecReceipt }); const server = CultMesh.createRudpDocumentServer("cultmesh-ts-codec-server", connectionId, { - documents: new CultNetDocumentRegistry([commandBinding, receiptBinding]), + documents: new CultNetDocumentRegistry([commandBinding, plainBinding, receiptBinding]), getCache: () => source.cache, bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, onDocumentPutRaw: async (document) => ({ @@ -97,6 +108,11 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r // A peer snapshot handle reads the same way, and accepts the snapshot by parsing it as the codec. assert.deepEqual(await CultMesh.documentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }).latest(), { noteId: "n1", body: "synced positional" }); + assert.deepEqual( + await CultMesh.documentFromPeerSnapshot(peer, codecNoteAlias, "n3", { timeoutMs: 1_000 }).latest(), + { noteId: "n3", body: "positional" }, + ); + const receipt = await CultMesh.publishRudpDocumentAndWaitForReceipt( "cultmesh-ts-codec-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`, commandBinding, "n2", { noteId: "n2", body: "command" }, receiptBinding, From f1e7b36a3996bf449bbd81436287e57de2c8811f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 10:25:29 +0200 Subject: [PATCH 27/44] Payload decode has one owner: cultcache-ts decodeDocumentPayload; wire values go through cultnet-ts/wire The RUDP put handler and the browser payload reader decoded a codec document with plain msgpack. cultcache-ts now exports decodeDocumentPayload (definition or plain), the RUDP server reads a put through the registry's definition, and decodeCultNetPayload takes the formatter. cultmesh-ts and cultmesh-browser no longer depend on @msgpack/msgpack: frames and commands use cultnet-ts/wire, and a test per package scans src and the manifest. The envelope probe fixture now runs the plain record first and last. Not yet built or tested on this commit; the verify job follows. Co-Authored-By: Claude Opus 5.5 --- package-lock.json | 2 -- packages/cultcache-ts/src/document.ts | 16 ++++++--- packages/cultcache-ts/src/index.ts | 2 +- .../test/decode-document-payload.test.ts | 33 +++++++++++++++++++ packages/cultmesh-browser/package.json | 1 - packages/cultmesh-browser/src/index.ts | 24 +++++++++----- .../test/cultmesh-browser.test.ts | 31 ++++++++++------- .../test/payload-decode-owner.test.ts | 29 ++++++++++++++++ packages/cultmesh-ts/package.json | 1 - packages/cultmesh-ts/src/index.ts | 19 +++++------ .../src/provider-rudp-transport.ts | 4 +-- .../src/provider-session-broker.ts | 6 ++-- .../cultmesh-ts/src/provider-session-wire.ts | 12 +++---- .../cultmesh-ts/test/codec-documents.test.ts | 26 +++++++++++---- packages/cultmesh-ts/test/cultmesh.test.ts | 28 ++++++++-------- .../test/payload-decode-owner.test.ts | 28 ++++++++++++++++ .../test/provider-rudp-transport.test.ts | 4 +-- .../rudp-document-server-send-failure.test.ts | 4 +-- packages/cultnet-ts/package.json | 5 +++ packages/cultnet-ts/src/wire-value.ts | 11 +++++++ 20 files changed, 211 insertions(+), 75 deletions(-) create mode 100644 packages/cultcache-ts/test/decode-document-payload.test.ts create mode 100644 packages/cultmesh-browser/test/payload-decode-owner.test.ts create mode 100644 packages/cultmesh-ts/test/payload-decode-owner.test.ts create mode 100644 packages/cultnet-ts/src/wire-value.ts diff --git a/package-lock.json b/package-lock.json index 5c0a5591..d07fbc78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1202,7 +1202,6 @@ "version": "0.3.0", "license": "MIT", "dependencies": { - "@msgpack/msgpack": "^3.1.3", "cultnet-ts": "^0.2.0" }, "devDependencies": { @@ -1218,7 +1217,6 @@ "license": "MIT", "dependencies": { "@gamecult/cultcache-ts": "^0.14.0", - "@msgpack/msgpack": "^3.1.2", "cultnet-ts": "^0.2.0", "koffi": "3.3.0" }, diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index a09fec81..e6cd080e 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -82,10 +82,18 @@ export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinit if (definition.codec && definition.formatter) { throw new Error(`CultCache document type "${definition.type}" declares both a codec and a formatter.`); } - return definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : { - encode: (value: unknown) => encode(value), - decode: (payload: Uint8Array) => decode(payload), - }); + return definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : plainDocumentFormatter); +} + +const plainDocumentFormatter: CultCacheDocumentFormatter = { + encode: (value: unknown) => encode(value), + decode: (payload: Uint8Array) => decode(payload), +}; + +// The one reader of a document payload outside the cache: through the definition's formatter, or as a plain msgpack +// value when the caller holds no definition for the schema. Every package above the cache reads a payload through here. +export function decodeDocumentPayload(definition: AnyCultCacheDocumentDefinition | undefined, payload: Uint8Array): unknown { + return (definition ? resolveDocumentFormatter(definition) : plainDocumentFormatter).decode(payload); } // The schema members of a document: the ones it declares, else those its codec derives, else none. diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 75a94f58..8fae6f4c 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,7 @@ export * as codecs from "./codec"; export type { Codec, SlotsCodec } from "./codec"; export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document"; +export { decodeDocumentPayload, defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; export type { AnyCultCacheDocumentDefinition, diff --git a/packages/cultcache-ts/test/decode-document-payload.test.ts b/packages/cultcache-ts/test/decode-document-payload.test.ts new file mode 100644 index 00000000..92fd2146 --- /dev/null +++ b/packages/cultcache-ts/test/decode-document-payload.test.ts @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { encode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { codecs, decodeDocumentPayload, defineDocumentType } from "../src"; + +const note = z.object({ id: z.string(), count: z.number() }); +const plain = defineDocumentType({ type: "t.plain", schemaId: "t.plain.v0", schema: note, name: "id" }); +const codec = defineDocumentType({ + type: "t.codec", + schemaId: "t.codec.v0", + schema: note, + name: "id", + codec: codecs.slots<{ id: string; count: number }>({ 0: ["id", codecs.string], 1: ["count", codecs.int32] }, "T.Note"), +}); +const formatted = defineDocumentType({ + type: "t.formatted", + schemaId: "t.formatted.v0", + schema: note, + name: "id", + formatter: { encode: (value) => encode([(value as { id: string }).id]), decode: () => ({ id: "from-formatter", count: 0 }) }, +}); + +test("a payload is read by its definition's formatter, a codec's, or as a plain value without a definition", () => { + assert.deepEqual(decodeDocumentPayload(undefined, encode({ id: "a", count: 1 })), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(plain, encode({ id: "a", count: 1 })), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(codec, encode(["a", 1])), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(formatted, new Uint8Array([0xc0])), { id: "from-formatter", count: 0 }); + // Without the definition the codec payload is its positional array, which is why a caller that holds one passes it. + assert.deepEqual(decodeDocumentPayload(undefined, encode(["a", 1])), ["a", 1]); +}); diff --git a/packages/cultmesh-browser/package.json b/packages/cultmesh-browser/package.json index 186ab630..e35f143f 100644 --- a/packages/cultmesh-browser/package.json +++ b/packages/cultmesh-browser/package.json @@ -40,7 +40,6 @@ "node": ">=20" }, "dependencies": { - "@msgpack/msgpack": "^3.1.3", "cultnet-ts": "^0.2.0" }, "devDependencies": { diff --git a/packages/cultmesh-browser/src/index.ts b/packages/cultmesh-browser/src/index.ts index ea37c17e..2aaca7bf 100644 --- a/packages/cultmesh-browser/src/index.ts +++ b/packages/cultmesh-browser/src/index.ts @@ -1,4 +1,4 @@ -import { decode, encode } from "@msgpack/msgpack"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { encodeCultNetMessageForWire, parseCultNetMessage, @@ -387,7 +387,7 @@ export class CultMeshBrowserClient implements AsyncDisposable { operation: options.operation, payloadSchema: options.payloadSchema, payloadEncoding: "messagepack-base64", - payload: bytesToBase64(encode(options.payload)), + payload: bytesToBase64(encodeWireValue(options.payload)), sourceRuntimeId: this.runtimeId, targetRuntimeId: this.identity.authorityRuntimeId, }; @@ -733,7 +733,7 @@ export class CultMeshBrowserClient implements AsyncDisposable { private send(message: CultNetMessage): void { const socket = this.#socket; if (!socket || socket.readyState !== 1) throw new Error("CultMesh browser route is not connected."); - const payload = encode(encodeCultNetMessageForWire(message)); + const payload = encodeWireValue(encodeCultNetMessageForWire(message)); if (payload.byteLength > this.#options.maxFrameBytes) { throw new Error(`CultMesh message exceeds the ${this.#options.maxFrameBytes}-byte limit.`); } @@ -760,11 +760,19 @@ export class CultMeshBrowserClient implements AsyncDisposable { } } -export function decodeCultNetPayload(record: CultNetRawDocumentRecord): T { +/** + * Reads a document record's payload. A document defined with a codec or a formatter is not a plain msgpack map: pass the + * formatter that reads it (a cultcache-ts `resolveDocumentFormatter(definition)`); without one the payload is read as a + * plain value, and a codec document comes back as its positional array. + */ +export function decodeCultNetPayload( + record: CultNetRawDocumentRecord, + formatter?: { decode(payload: Uint8Array): unknown }, +): T { if (record.payloadEncoding !== "messagepack") { throw new Error(`Unsupported CultNet document payload '${record.payloadEncoding}'.`); } - return decode(record.payload) as T; + return (formatter ? formatter.decode(record.payload) : decodeWireValue(record.payload)) as T; } function recordMatchesSchema(record: CultNetRawDocumentRecord, requestedSchema: string): boolean { @@ -777,7 +785,7 @@ export function decodeCultNetOperationPayload(response: CultNetOperationRespo if (response.payloadEncoding !== "messagepack-base64") { throw new Error(`Unsupported CultNet operation payload '${response.payloadEncoding ?? "unspecified"}'.`); } - return decode(base64ToBytes(response.payload)) as T; + return decodeWireValue(base64ToBytes(response.payload)) as T; } function requireWebSocketEndpoint(value: string, field: string): URL { @@ -794,7 +802,7 @@ function requireWebSocketEndpoint(value: string, field: string): URL { } function sendSocketMessage(socket: CultMeshBrowserSocket, message: CultNetMessage, maxFrameBytes: number): void { - const payload = encode(encodeCultNetMessageForWire(message)); + const payload = encodeWireValue(encodeCultNetMessageForWire(message)); if (payload.byteLength > maxFrameBytes) { throw new Error(`CultNet message exceeds the ${maxFrameBytes}-byte limit.`); } @@ -815,7 +823,7 @@ async function decodeSocketMessage(event: MessageEvent, maxFrameBytes: number): if (payload.byteLength > maxFrameBytes) { throw new Error(`CultNet frame exceeds the ${maxFrameBytes}-byte limit.`); } - return parseCultNetMessage(decode(payload)); + return parseCultNetMessage(decodeWireValue(payload)); } async function selectOdinRoute( diff --git a/packages/cultmesh-browser/test/cultmesh-browser.test.ts b/packages/cultmesh-browser/test/cultmesh-browser.test.ts index 59f9fac1..61b560c2 100644 --- a/packages/cultmesh-browser/test/cultmesh-browser.test.ts +++ b/packages/cultmesh-browser/test/cultmesh-browser.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decode, encode } from "@msgpack/msgpack"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { encodeCultNetMessageForWire, parseCultNetMessage, @@ -372,7 +372,7 @@ test("browser client replays one idempotent operation after route loss without d const open = () => { let socket: FakeSocket; socket = new FakeSocket(wire => { - const message = parseCultNetMessage(decode(wire)); + const message = parseCultNetMessage(decodeWireValue(wire)); if (message.schemaVersion === "cultmesh.session_open.v2") { socket.deliver({ schemaVersion: "cultmesh.session_accepted.v2", @@ -400,7 +400,7 @@ test("browser client replays one idempotent operation after route loss without d status: "accepted", payloadSchema: "sample.increment_receipt.v1", payloadEncoding: "messagepack-base64", - payload: Buffer.from(encode({ count: effectCount })).toString("base64"), + payload: Buffer.from(encodeWireValue({ count: effectCount })).toString("base64"), sourceRuntimeId: "sample.counter-daemon", }; results.set(message.messageId, response!); @@ -600,7 +600,7 @@ class FakeProvider { } private receive(socket: FakeSocket, wire: Uint8Array): void { - const message = parseCultNetMessage(decode(wire)); + const message = parseCultNetMessage(decodeWireValue(wire)); switch (message.schemaVersion) { case "cultmesh.session_open.v2": this.acceptSession(socket, message); @@ -654,7 +654,7 @@ class FakeProvider { status: "invalid", payloadSchema: "gamecult.cultnet.operation_failure.v1", payloadEncoding: "messagepack-base64", - payload: bytesToBase64(encode({ + payload: bytesToBase64(encodeWireValue({ code: "request-schema-mismatch", message: "Expected payload schema 'sample.increment.v1'.", })), @@ -663,7 +663,7 @@ class FakeProvider { }); return; } - const payload = decode(base64ToBytes(message.payload)) as { amount: number }; + const payload = decodeWireValue(base64ToBytes(message.payload)) as { amount: number }; this.#count += payload.amount; for (const [peer, subscriptions] of this.#subscriptions) { for (const subscriptionId of subscriptions.keys()) { @@ -684,7 +684,7 @@ class FakeProvider { status: "accepted", payloadSchema: "sample.increment_receipt.v1", payloadEncoding: "messagepack-base64", - payload: bytesToBase64(encode({ count: this.#count })), + payload: bytesToBase64(encodeWireValue({ count: this.#count })), sourceRuntimeId: this.#responseRuntimeId, }); } @@ -698,7 +698,7 @@ class FakeProvider { recordKey: "counter:main", storedAt: "2026-08-17T00:00:00Z", payloadEncoding: "messagepack", - payload: encode({ count: this.#count }), + payload: encodeWireValue({ count: this.#count }), sourceRuntimeId: this.#runtimeId, }; } @@ -716,7 +716,7 @@ class FakeOdin { open(): FakeSocket { const socket = new FakeSocket(wire => { - const message = parseCultNetMessage(decode(wire)); + const message = parseCultNetMessage(decodeWireValue(wire)); if (message.schemaVersion !== "cultmesh.verse_catalog_request.v0") return; socket.deliver({ schemaVersion: "cultmesh.verse_catalog_response.v0", @@ -785,7 +785,7 @@ function routeForCatalog(route: CultMeshBrowserRoute): NonNullable< function handshakeOnlySocket(route: CultMeshBrowserRoute): FakeSocket { let socket: FakeSocket; socket = new FakeSocket(wire => { - const message = parseCultNetMessage(decode(wire)); + const message = parseCultNetMessage(decodeWireValue(wire)); if (message.schemaVersion !== "cultmesh.session_open.v2") return; socket.deliver({ schemaVersion: "cultmesh.session_accepted.v2", @@ -850,7 +850,7 @@ class FakeSocket implements CultMeshBrowserSocket { } deliver(message: CultNetMessage): void { - const wire = encode(encodeCultNetMessageForWire(message)); + const wire = encodeWireValue(encodeCultNetMessageForWire(message)); queueMicrotask(() => this.onmessage?.({ data: wire.slice().buffer } as MessageEvent)); } } @@ -899,7 +899,7 @@ async function createSignedRoute(endpoint: string): Promise<{ function signedHandshakeSocket(route: CultMeshBrowserRoute, providerPrivate: CryptoKey): FakeSocket { let socket: FakeSocket; socket = new FakeSocket(wire => { - const message = parseCultNetMessage(decode(wire)); + const message = parseCultNetMessage(decodeWireValue(wire)); if (message.schemaVersion !== "cultmesh.session_open.v2") return; void crypto.subtle.sign( { name: "ECDSA", hash: "SHA-256" }, @@ -975,3 +975,10 @@ function base64ToBytes(value: string): Uint8Array { const binary = atob(value); return Uint8Array.from(binary, character => character.charCodeAt(0)); } + +test("decodeCultNetPayload reads a payload through the formatter it is given, and plain without one", () => { + const record = { schemaId: "t.codec.v0", recordKey: "k", payloadEncoding: "messagepack", payload: encodeWireValue([7]) } as unknown as CultNetRawDocumentRecord; + const formatter = { decode: (payload: Uint8Array) => ({ count: (decodeWireValue(payload) as number[])[0] }) }; + assert.deepEqual(decodeCultNetPayload<{ count: number }>(record, formatter), { count: 7 }); + assert.deepEqual(decodeCultNetPayload(record), [7]); +}); diff --git a/packages/cultmesh-browser/test/payload-decode-owner.test.ts b/packages/cultmesh-browser/test/payload-decode-owner.test.ts new file mode 100644 index 00000000..5ed8f915 --- /dev/null +++ b/packages/cultmesh-browser/test/payload-decode-owner.test.ts @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +// A document payload is decoded by cultcache-ts alone (decodeDocumentPayload); every other value on a wire goes through +// cultnet-ts's decodeWireValue / encodeWireValue. So no file of this package imports msgpack, and the manifest does not +// list it, which keeps a new direct decode from arriving unnoticed. +// Limits: a text scan. It reads the package's src files for the module specifier and the manifest for the dependency +// name. It does not follow a re-export from another package, a specifier assembled at run time, or a hand-written +// decoder; the dependency absence is what makes the specifier unresolvable here when the workspace does not hoist it. +const root = fileURLToPath(new URL("../../", import.meta.url)); + +function sourceFiles(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => + entry.isDirectory() ? sourceFiles(join(dir, entry.name)) : entry.name.endsWith(".ts") ? [join(dir, entry.name)] : []); +} + +test("no source file imports msgpack and the manifest does not list it", () => { + const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as Record | undefined>; + for (const section of ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"]) { + assert.equal(manifest[section]?.["@msgpack/msgpack"], undefined, section); + } + const files = sourceFiles(join(root, "src")); + assert.ok(files.length > 5, "fixture: the scan reads the package's sources"); + const offenders = files.filter((file) => /@msgpack\/msgpack/.test(readFileSync(file, "utf8"))); + assert.deepEqual(offenders, []); +}); diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index 4d59133a..b53555be 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -43,7 +43,6 @@ "node": ">=20" }, "dependencies": { - "@msgpack/msgpack": "^3.1.2", "@gamecult/cultcache-ts": "^0.14.0", "cultnet-ts": "^0.2.0", "koffi": "3.3.0" diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 9c77e586..f738cdcd 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -1,5 +1,5 @@ import { createSocket, type Socket } from "node:dgram"; -import { decode, encode } from "@msgpack/msgpack"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { CultCache, @@ -8,7 +8,7 @@ import { type CacheBackingStore, type CultCacheEnvelope, type CultCacheDocumentValue, - resolveDocumentFormatter, + decodeDocumentPayload, } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, @@ -5957,7 +5957,7 @@ export class CultMesh { payload: Uint8Array, ): Promise { if (record.closed) return; - const message = parseCultNetMessage(decode(payload), wireContract); + const message = parseCultNetMessage(decodeWireValue(payload), wireContract); switch (message.schemaVersion) { case "cultnet.snapshot_request.v0": { if (!options.getCache) { @@ -6000,7 +6000,7 @@ export class CultMesh { sendSchemaMessage(record, { schemaVersion: "cultnet.error.v0", error: unservable.message }); return; } - const receipt = await options.onDocumentPutRaw(normalizeRudpDocumentPut(message, record.remote)); + const receipt = await options.onDocumentPutRaw(normalizeRudpDocumentPut(message, record.remote, options.documents)); if (receipt) { try { sendSchemaMessage(record, receiptMessage(receipt, message.messageId)); @@ -6138,7 +6138,7 @@ export class CultMesh { /** The one reply encoder, shared by sends and by put admission. */ function encodeSchemaMessage(message: CultNetMessage): Uint8Array { - return encode(encodeCultNetMessageForWire(message, wireContract)); + return encodeWireValue(encodeCultNetMessageForWire(message, wireContract)); } /** Builds the receipt `onDocumentPutRaw` returned, under the put's message id by default. */ @@ -6663,6 +6663,7 @@ function requireNonEmpty(value: string, name: string): void { function normalizeRudpDocumentPut( message: CultNetMessage, remote: { address: string; family: string; port: number }, + documents: CultNetDocumentRegistry, ): CultMeshRudpDocumentPut { if (message.schemaVersion !== "cultnet.document_put_raw.v0") { throw new Error(`Expected cultnet.document_put_raw.v0, received ${message.schemaVersion}.`); @@ -6678,7 +6679,7 @@ function normalizeRudpDocumentPut( schemaId: document.schemaId, recordKey: document.recordKey, storedAt: document.storedAt ?? new Date().toISOString(), - payload: decode(document.payload), + payload: decodeDocumentPayload(documents.getBySchemaId(document.schemaId)?.definition, document.payload), sourceRuntimeId: document.sourceRuntimeId ?? null, sourceAgentId: document.sourceAgentId ?? null, sourceRole: document.sourceRole ?? null, @@ -6777,11 +6778,9 @@ async function resolveCultNetPeer( } // A document payload is read the way its definition's cache reads it (a codec or formatter document is not a plain -// msgpack map); a payload with no definition is plain msgpack. +// msgpack map); a payload with no definition is plain msgpack. Both are cultcache-ts's decision. function decodeCultMeshPayload(schemaOrDefinition: string | AnyCultCacheDocumentDefinition, payload: Uint8Array): unknown { - return typeof schemaOrDefinition === "string" - ? decode(payload) - : resolveDocumentFormatter(schemaOrDefinition).decode(payload); + return decodeDocumentPayload(typeof schemaOrDefinition === "string" ? undefined : schemaOrDefinition, payload); } function decodeCultNetRawDocumentPayload( diff --git a/packages/cultmesh-ts/src/provider-rudp-transport.ts b/packages/cultmesh-ts/src/provider-rudp-transport.ts index 0557bf48..7e94df02 100644 --- a/packages/cultmesh-ts/src/provider-rudp-transport.ts +++ b/packages/cultmesh-ts/src/provider-rudp-transport.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { createSocket, type Socket } from "node:dgram"; -import { encode } from "@msgpack/msgpack"; +import { encodeWireValue } from "cultnet-ts/wire"; import { CultNetPeer, CultNetRudpSocketTransportConnection, @@ -171,7 +171,7 @@ class RudpProviderConnection implements CultMeshProviderConnection { recordKey: publication.recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encode(publication.value), + payload: encodeWireValue(publication.value), sourceRuntimeId: this.#identity.serviceInstanceId, sourceAgentId: this.#identity.providerId, sourceRole: "provider", diff --git a/packages/cultmesh-ts/src/provider-session-broker.ts b/packages/cultmesh-ts/src/provider-session-broker.ts index eb9e18b5..8f650e36 100644 --- a/packages/cultmesh-ts/src/provider-session-broker.ts +++ b/packages/cultmesh-ts/src/provider-session-broker.ts @@ -1,4 +1,4 @@ -import { encode } from "@msgpack/msgpack"; +import { encodeWireValue } from "cultnet-ts/wire"; import type { CultNetDocumentPutRawMessage, CultNetOperationRequestMessage, @@ -420,7 +420,7 @@ export class CultMeshProviderSessionBroker { recordKey: `${command.providerId}/${command.serviceInstanceId}/${command.commandId}`, storedAt: this.#clock.now().toISOString(), payloadEncoding: "messagepack", - payload: encode(command), + payload: encodeWireValue(command), sourceRuntimeId: this.#options.runtimeId, sourceRole: "provider-session-broker", }, @@ -513,7 +513,7 @@ function documentTuple(document: Pick(payload: string): T { if (typeof payload !== "string" || payload.length === 0) { throw new Error("CultMesh provider-session payload must be non-empty MessagePack base64."); } - return decode(Buffer.from(payload, "base64")) as T; + return decodeWireValue(Buffer.from(payload, "base64")) as T; } export function encodeProviderConnectEvidence( evidence: CultMeshProviderConnectEvidenceWire, ): Uint8Array { assertProviderConnectEvidence(evidence); - return encode(evidence); + return encodeWireValue(evidence); } export function decodeProviderConnectEvidence( payload: Uint8Array, ): CultMeshProviderConnectEvidenceWire { - const evidence = decode(payload) as unknown; + const evidence = decodeWireValue(payload) as unknown; assertProviderConnectEvidence(evidence); return evidence; } @@ -162,7 +162,7 @@ export function decodeProviderCommandDocument(document: CultNetRawDocumentRecord if (document.payloadEncoding !== "messagepack") { throw new Error(`Provider command ${document.recordKey} must use MessagePack.`); } - const command = decode(document.payload) as Partial; + const command = decodeWireValue(document.payload) as Partial; requireText(command.commandId, "commandId"); requireText(command.commandKind, "commandKind"); requireText(command.providerId, "providerId"); diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index 4a0d2494..aeb2a14d 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -63,12 +63,16 @@ test("the mesh reads a stored codec document as the cache does, and a plain docu assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" }); assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" }); - // A store holding a plain record first and a codec record second: the alias, found by parsing, takes the codec one. - const bothPath = join(dir, "both.ccmp"); - const bothNode = await CultMesh.startNode(bothPath, { documents: [plainNote, codecNote] }); - await bothNode.put(plainNote, "n2", { noteId: "n2", body: "map" }); - await bothNode.put(codecNote, "n1", { noteId: "n1", body: "positional" }); - assert.deepEqual(await cultMeshDocumentFromSingleFile(bothPath, codecNoteAlias).latest(), { noteId: "n1", body: "positional" }); + // A store holding a plain record and a codec record, the alias found by parsing takes the codec one whichever the + // single-file store (which sorts records by key) puts first: the probe has to refuse the plain record, not meet it + // second. + for (const [plainKey, codecKey] of [["a-plain", "z-codec"], ["z-plain", "a-codec"]] as const) { + const bothPath = join(dir, `both-${plainKey}.ccmp`); + const bothNode = await CultMesh.startNode(bothPath, { documents: [plainNote, codecNote] }); + await bothNode.put(plainNote, plainKey, { noteId: plainKey, body: "map" }); + await bothNode.put(codecNote, codecKey, { noteId: codecKey, body: "positional" }); + assert.deepEqual(await cultMeshDocumentFromSingleFile(bothPath, codecNoteAlias).latest(), { noteId: codecKey, body: "positional" }); + } assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" }); }); @@ -85,11 +89,12 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r const commandBinding = defineCultNetDocumentBinding({ definition: codecNote }); const plainBinding = defineCultNetDocumentBinding({ definition: plainNote }); const receiptBinding = defineCultNetDocumentBinding({ definition: codecReceipt }); + const puts: unknown[] = []; const server = CultMesh.createRudpDocumentServer("cultmesh-ts-codec-server", connectionId, { documents: new CultNetDocumentRegistry([commandBinding, plainBinding, receiptBinding]), getCache: () => source.cache, bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, - onDocumentPutRaw: async (document) => ({ + onDocumentPutRaw: async (document) => (puts.push(document.payload), { binding: receiptBinding, recordKey: document.recordKey, value: { commandId: document.recordKey, body: "applied" }, @@ -119,6 +124,13 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r { messageId: "n2", receiptTimeoutMs: 1_000, resendPollMs: 5 }, ); assert.deepEqual(receipt, { commandId: "n2", body: "applied" }); + // The handler is handed the document, not its wire shape: the server reads a put through the registered definition. + await CultMesh.publishRudpDocumentAndWaitForReceipt( + "cultmesh-ts-plain-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`, + plainBinding, "n4", { noteId: "n4", body: "plain command" }, receiptBinding, + { messageId: "n4", receiptTimeoutMs: 1_000, resendPollMs: 5 }, + ); + assert.deepEqual(puts, [{ noteId: "n2", body: "command" }, { noteId: "n4", body: "plain command" }]); } finally { peer?.close(); server.close(); diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 5ec961a5..4cdce8df 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; -import { decode, encode } from "@msgpack/msgpack"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { z } from "zod"; import { defineDocumentType } from "@gamecult/cultcache-ts"; import { @@ -268,7 +268,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa type: "runtime.generated.incompatible-note", schemaId: "runtime.generated.incompatible-note.v1", storedAt: "2026-06-27T00:00:00Z", - payload: encode({ nope: "not the requested shape" }), + payload: encodeWireValue({ nope: "not the requested shape" }), catalogEntry: { schemaId: "runtime.generated.incompatible-note.v1", schemaName: "runtime.generated.incompatible-note", @@ -283,7 +283,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa type: "runtime.generated.cultmesh.note", schemaId: "runtime.generated.cultmesh.note.ui.42", storedAt: "2026-06-27T00:00:01Z", - payload: encode({ + payload: encodeWireValue({ noteId: "note:foreign", body: "foreign schema id, local store shape", }), @@ -2820,7 +2820,7 @@ test("CultMesh TS RUDP document server accepts raw document puts from multiple p recordKey: "note:a", storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encode("first"), + payload: encodeWireValue("first"), sourceRuntimeId: "client-a", }, }); @@ -2832,7 +2832,7 @@ test("CultMesh TS RUDP document server accepts raw document puts from multiple p recordKey: "note:b", storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encode("second"), + payload: encodeWireValue("second"), sourceRuntimeId: "client-b", }, }); @@ -3311,7 +3311,7 @@ function notePut( recordKey, storedAt, payloadEncoding: "messagepack", - payload: encode({ noteId: recordKey, body: "x".repeat(bodyLength) }), + payload: encodeWireValue({ noteId: recordKey, body: "x".repeat(bodyLength) }), ...(sourceRuntimeId ? { sourceRuntimeId } : {}), }, }; @@ -3324,7 +3324,7 @@ function notePut( */ function servedAloneBytes(put: CultNetDocumentPutRawMessage): number { const { schemaId, recordKey, storedAt, payloadEncoding, payload } = put.document; - return encode({ + return encodeWireValue({ schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: "0", documents: [{ schemaId, recordKey, storedAt, payloadEncoding, payload }], @@ -3337,7 +3337,7 @@ function servedAloneBytes(put: CultNetDocumentPutRawMessage): number { * asked of the server, so a server that sized another receipt is caught. */ function echoReceiptBytes(put: CultNetDocumentPutRawMessage): number { - return encode({ + return encodeWireValue({ schemaVersion: "cultnet.document_put_raw.v0", messageId: put.messageId, document: { @@ -3345,7 +3345,7 @@ function echoReceiptBytes(put: CultNetDocumentPutRawMessage): number { recordKey: put.document.recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encode(decode(put.document.payload)), + payload: encodeWireValue(decodeWireValue(put.document.payload)), sourceRuntimeId: SERVED_BOUND_SERVER, sourceAgentId: undefined, sourceRole: "provider", @@ -3397,7 +3397,7 @@ async function withServedBoundRig( recordKey: document.recordKey, storedAt: document.storedAt, payloadEncoding: "messagepack", - payload: encode(document.payload), + payload: encodeWireValue(document.payload), }, }); if (!handler.receipt) return undefined; @@ -3448,7 +3448,7 @@ function servedBodies(reply: CultNetMessage): Array<[string, number]> { assert.equal(reply.schemaVersion, "cultnet.snapshot_response_raw.v0"); return (reply as CultNetSnapshotResponseRawMessage).documents.map((document) => [ document.recordKey, - (decode(document.payload) as { body: string }).body.length, + (decodeWireValue(document.payload) as { body: string }).body.length, ]); } @@ -3567,7 +3567,7 @@ async function withRawPeerRig( const request = (messageId: string, recordKeys: string[]): number[] => { const packets = session.sendMany( "schema", - encode({ schemaVersion: "cultnet.snapshot_request.v0", messageId, recordKeys }), + encodeWireValue({ schemaVersion: "cultnet.snapshot_request.v0", messageId, recordKeys }), { reliable: true, ordered: true, nowMs: 2, maxFragmentBytes: 1200 }, ); packets.forEach(send); @@ -3586,7 +3586,7 @@ function refusalIn(packets: CultNetRudpPacket[]): { message: Record packet.packetType === "data" && packet.channelId === "schema" && !packet.reliable) .sort((a, b) => (a.fragmentIndex ?? 0) - (b.fragmentIndex ?? 0)); if (parts.length === 0) return undefined; - const message = decode(Buffer.concat(parts.map((part) => Buffer.from(part.payload ?? new Uint8Array())))) as Record; + const message = decodeWireValue(Buffer.concat(parts.map((part) => Buffer.from(part.payload ?? new Uint8Array())))) as Record; return { message, parts }; } @@ -3655,7 +3655,7 @@ test("CultMesh TS RUDP document server honours maxPayloadBytes for replies and j // A put whose own frame is over the limit only by the provenance it carries is served at the // bound, so it is admitted and served. const carried = notePut("put-at", "note:at", atBound, `runtime-${"r".repeat(200)}`); - assert.ok(encode(carried).byteLength > limit, "fixture: the put's frame is over the limit"); + assert.ok(encodeWireValue(carried).byteLength > limit, "fixture: the put's frame is over the limit"); rig.peer.send(carried); await waitForAdmission(rig, "note:at"); assert.deepEqual(servedBodies(await snapshotUnderShortestId(rig, "note:at")), [["note:at", atBound]]); diff --git a/packages/cultmesh-ts/test/payload-decode-owner.test.ts b/packages/cultmesh-ts/test/payload-decode-owner.test.ts new file mode 100644 index 00000000..e267a9c2 --- /dev/null +++ b/packages/cultmesh-ts/test/payload-decode-owner.test.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; + +// A document payload is decoded by cultcache-ts alone (decodeDocumentPayload); every other value on a wire goes through +// cultnet-ts's decodeWireValue / encodeWireValue. So no file of this package imports msgpack, and the manifest does not +// list it, which keeps a new direct decode from arriving unnoticed. +// Limits: a text scan. It reads the package's src files for the module specifier and the manifest for the dependency +// name. It does not follow a re-export from another package, a specifier assembled at run time, or a hand-written +// decoder; the dependency absence is what makes the specifier unresolvable here when the workspace does not hoist it. +const root = join(__dirname, "..", ".."); + +function sourceFiles(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => + entry.isDirectory() ? sourceFiles(join(dir, entry.name)) : entry.name.endsWith(".ts") ? [join(dir, entry.name)] : []); +} + +test("no source file imports msgpack and the manifest does not list it", () => { + const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as Record | undefined>; + for (const section of ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"]) { + assert.equal(manifest[section]?.["@msgpack/msgpack"], undefined, section); + } + const files = sourceFiles(join(root, "src")); + assert.ok(files.length > 5, "fixture: the scan reads the package's sources"); + const offenders = files.filter((file) => /@msgpack\/msgpack/.test(readFileSync(file, "utf8"))); + assert.deepEqual(offenders, []); +}); diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index f1dd5d2d..abb8f010 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createSocket } from "node:dgram"; -import { decode } from "@msgpack/msgpack"; +import { decodeWireValue } from "cultnet-ts/wire"; import { CultNetDocumentRegistry, CultNetRudpSession, @@ -46,7 +46,7 @@ test("RUDP provider transport requires broker acceptance for the full lifecycle" session.connectPayload !== undefined && decodeProviderConnectEvidence(session.connectPayload).sessionToken === "signed-provider-token", acceptPublication: (_identity, publicationId, document) => { - accepted.set(publicationId, decode(document.payload)); + accepted.set(publicationId, decodeWireValue(document.payload)); acceptedProvenance.set(publicationId, { runtime: document.sourceRuntimeId, agent: document.sourceAgentId, diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts index 64f19111..8d0425e4 100644 --- a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -5,7 +5,7 @@ import assert from "node:assert/strict"; import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; import test from "node:test"; -import { encode } from "@msgpack/msgpack"; +import { encodeWireValue } from "cultnet-ts/wire"; import { CultNetDocumentRegistry, CultNetRudpSession, @@ -43,7 +43,7 @@ function put(recordKey: string) { recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack" as const, - payload: encode(recordKey), + payload: encodeWireValue(recordKey), }, }; } diff --git a/packages/cultnet-ts/package.json b/packages/cultnet-ts/package.json index e72a10aa..9147d4ea 100644 --- a/packages/cultnet-ts/package.json +++ b/packages/cultnet-ts/package.json @@ -15,6 +15,11 @@ "require": "./dist/contracts.js", "default": "./dist/contracts.js" }, + "./wire": { + "types": "./dist/wire-value.d.ts", + "require": "./dist/wire-value.js", + "default": "./dist/wire-value.js" + }, "./idunn-runtime": { "types": "./dist/idunn-runtime-authority.d.ts", "require": "./dist/idunn-runtime-authority.js", diff --git a/packages/cultnet-ts/src/wire-value.ts b/packages/cultnet-ts/src/wire-value.ts new file mode 100644 index 00000000..43a5e029 --- /dev/null +++ b/packages/cultnet-ts/src/wire-value.ts @@ -0,0 +1,11 @@ +import { decode, encode } from "@msgpack/msgpack"; + +// The one msgpack entry point above the wire for packages that do not own it: a frame, an envelope, a command, any +// value that is not a document payload. A document payload is read through cultcache-ts's decodeDocumentPayload. +export function encodeWireValue(value: unknown): Uint8Array { + return encode(value); +} + +export function decodeWireValue(bytes: ArrayLike | ArrayBufferView | ArrayBuffer): unknown { + return decode(bytes); +} From 294469e3009fc3e94783c8be7ae042ab66e14aff Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 10:31:08 +0200 Subject: [PATCH 28/44] Payload-owner guard finds its package root from the file; refusals pinned in the decode tests The guard test counted its package root from the compiled layout and failed under tsx and for a one-file src; it now walks up to the package.json. The browser's non-messagepack refusal and the cache's codec-and-formatter refusal are pinned by message. Co-Authored-By: Claude Opus 5.5 --- .../test/decode-document-payload.test.ts | 6 ++++++ .../cultmesh-browser/test/cultmesh-browser.test.ts | 5 +++++ .../test/payload-decode-owner.test.ts | 12 ++++++++---- .../cultmesh-ts/test/payload-decode-owner.test.ts | 12 ++++++++---- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/cultcache-ts/test/decode-document-payload.test.ts b/packages/cultcache-ts/test/decode-document-payload.test.ts index 92fd2146..a63859ee 100644 --- a/packages/cultcache-ts/test/decode-document-payload.test.ts +++ b/packages/cultcache-ts/test/decode-document-payload.test.ts @@ -31,3 +31,9 @@ test("a payload is read by its definition's formatter, a codec's, or as a plain // Without the definition the codec payload is its positional array, which is why a caller that holds one passes it. assert.deepEqual(decodeDocumentPayload(undefined, encode(["a", 1])), ["a", 1]); }); + +test("a definition that declares both a codec and a formatter is refused where its payload is read", () => { + const both = { ...codec, formatter: formatted.formatter } as typeof codec; + assert.throws(() => decodeDocumentPayload(both, encode(["a", 1])), (error: Error) => + error.message === 'CultCache document type "t.codec" declares both a codec and a formatter.'); +}); diff --git a/packages/cultmesh-browser/test/cultmesh-browser.test.ts b/packages/cultmesh-browser/test/cultmesh-browser.test.ts index 61b560c2..fb65fbf3 100644 --- a/packages/cultmesh-browser/test/cultmesh-browser.test.ts +++ b/packages/cultmesh-browser/test/cultmesh-browser.test.ts @@ -982,3 +982,8 @@ test("decodeCultNetPayload reads a payload through the formatter it is given, an assert.deepEqual(decodeCultNetPayload<{ count: number }>(record, formatter), { count: 7 }); assert.deepEqual(decodeCultNetPayload(record), [7]); }); + +test("decodeCultNetPayload refuses a payload that is not messagepack, naming the encoding and not the payload", () => { + const record = { schemaId: "t.v0", recordKey: "k", payloadEncoding: "json", payload: encodeWireValue("secret-value") } as unknown as CultNetRawDocumentRecord; + assert.throws(() => decodeCultNetPayload(record), (error: Error) => error.message === "Unsupported CultNet document payload 'json'."); +}); diff --git a/packages/cultmesh-browser/test/payload-decode-owner.test.ts b/packages/cultmesh-browser/test/payload-decode-owner.test.ts index 5ed8f915..e5a23f78 100644 --- a/packages/cultmesh-browser/test/payload-decode-owner.test.ts +++ b/packages/cultmesh-browser/test/payload-decode-owner.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; -import { readdirSync, readFileSync } from "node:fs"; -import { join } from "node:path"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; @@ -10,7 +10,11 @@ import test from "node:test"; // Limits: a text scan. It reads the package's src files for the module specifier and the manifest for the dependency // name. It does not follow a re-export from another package, a specifier assembled at run time, or a hand-written // decoder; the dependency absence is what makes the specifier unresolvable here when the workspace does not hoist it. -const root = fileURLToPath(new URL("../../", import.meta.url)); +// The package root: the nearest directory above this file (compiled or run in place) that holds a package.json. +function packageRoot(from: string): string { + return existsSync(join(from, "package.json")) ? from : packageRoot(dirname(from)); +} +const root = packageRoot(dirname(fileURLToPath(import.meta.url))); function sourceFiles(dir: string): string[] { return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => @@ -23,7 +27,7 @@ test("no source file imports msgpack and the manifest does not list it", () => { assert.equal(manifest[section]?.["@msgpack/msgpack"], undefined, section); } const files = sourceFiles(join(root, "src")); - assert.ok(files.length > 5, "fixture: the scan reads the package's sources"); + assert.ok(files.length >= 1, "fixture: the scan reads the package's sources"); const offenders = files.filter((file) => /@msgpack\/msgpack/.test(readFileSync(file, "utf8"))); assert.deepEqual(offenders, []); }); diff --git a/packages/cultmesh-ts/test/payload-decode-owner.test.ts b/packages/cultmesh-ts/test/payload-decode-owner.test.ts index e267a9c2..f6471bb4 100644 --- a/packages/cultmesh-ts/test/payload-decode-owner.test.ts +++ b/packages/cultmesh-ts/test/payload-decode-owner.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; -import { readdirSync, readFileSync } from "node:fs"; -import { join } from "node:path"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import test from "node:test"; // A document payload is decoded by cultcache-ts alone (decodeDocumentPayload); every other value on a wire goes through @@ -9,7 +9,11 @@ import test from "node:test"; // Limits: a text scan. It reads the package's src files for the module specifier and the manifest for the dependency // name. It does not follow a re-export from another package, a specifier assembled at run time, or a hand-written // decoder; the dependency absence is what makes the specifier unresolvable here when the workspace does not hoist it. -const root = join(__dirname, "..", ".."); +// The package root: the nearest directory above this file (compiled or run in place) that holds a package.json. +function packageRoot(from: string): string { + return existsSync(join(from, "package.json")) ? from : packageRoot(dirname(from)); +} +const root = packageRoot(__dirname); function sourceFiles(dir: string): string[] { return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => @@ -22,7 +26,7 @@ test("no source file imports msgpack and the manifest does not list it", () => { assert.equal(manifest[section]?.["@msgpack/msgpack"], undefined, section); } const files = sourceFiles(join(root, "src")); - assert.ok(files.length > 5, "fixture: the scan reads the package's sources"); + assert.ok(files.length >= 1, "fixture: the scan reads the package's sources"); const offenders = files.filter((file) => /@msgpack\/msgpack/.test(readFileSync(file, "utf8"))); assert.deepEqual(offenders, []); }); From a3a45c79ebf2ab1ece3391113cb8f5eec607b2de Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 19:21:32 +0200 Subject: [PATCH 29/44] Delete the text-scan payload guards, the schemaId overloads and the definition-less decode door The two payload-decode-owner text-scan tests go, as do the string-schemaId overloads and branches of the cultMeshDocumentFrom* functions and statics, decodeCultMeshPayload, decodeCultNetRawDocumentPayload, toUint8Array, replication's second decodeDocumentValue, and the undefined branch of decodeDocumentPayload. decodeDocumentPayload now requires the definition and also runs its schema (the one reader). A publication binding names a definition, never a bare schema id; the RUDP put handler refuses a schema with no registered definition instead of plain-decoding it. Builds in every package; tests not yet rerun. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/src/document.ts | 12 +- .../test/decode-document-payload.test.ts | 5 +- .../test/payload-decode-owner.test.ts | 33 -- packages/cultmesh-ts/src/index.ts | 464 ++++-------------- packages/cultmesh-ts/test/cultmesh.test.ts | 7 +- .../test/payload-decode-owner.test.ts | 32 -- packages/cultnet-ts/src/replication.ts | 15 +- 7 files changed, 105 insertions(+), 463 deletions(-) delete mode 100644 packages/cultmesh-browser/test/payload-decode-owner.test.ts delete mode 100644 packages/cultmesh-ts/test/payload-decode-owner.test.ts diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index e6cd080e..50e08afb 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -7,6 +7,7 @@ import type { CultCacheSchemaCatalogMember, CultCacheDocumentDefinition, CultCacheDocumentRegistry, + CultCacheDocumentValue, CultCacheSchema, } from "./types"; @@ -90,10 +91,13 @@ const plainDocumentFormatter: CultCacheDocumentFormatter = { decode: (payload: Uint8Array) => decode(payload), }; -// The one reader of a document payload outside the cache: through the definition's formatter, or as a plain msgpack -// value when the caller holds no definition for the schema. Every package above the cache reads a payload through here. -export function decodeDocumentPayload(definition: AnyCultCacheDocumentDefinition | undefined, payload: Uint8Array): unknown { - return (definition ? resolveDocumentFormatter(definition) : plainDocumentFormatter).decode(payload); +// The one reader of a document payload: through the definition's formatter, codec or plain map, then its schema. A +// reader holds the definition or it has no business reading the bytes, so there is no definition-less form. +export function decodeDocumentPayload( + definition: TDefinition, + payload: Uint8Array, +): CultCacheDocumentValue { + return definition.schema.parse(resolveDocumentFormatter(definition).decode(payload)) as CultCacheDocumentValue; } // The schema members of a document: the ones it declares, else those its codec derives, else none. diff --git a/packages/cultcache-ts/test/decode-document-payload.test.ts b/packages/cultcache-ts/test/decode-document-payload.test.ts index a63859ee..e212c54a 100644 --- a/packages/cultcache-ts/test/decode-document-payload.test.ts +++ b/packages/cultcache-ts/test/decode-document-payload.test.ts @@ -23,13 +23,10 @@ const formatted = defineDocumentType({ formatter: { encode: (value) => encode([(value as { id: string }).id]), decode: () => ({ id: "from-formatter", count: 0 }) }, }); -test("a payload is read by its definition's formatter, a codec's, or as a plain value without a definition", () => { - assert.deepEqual(decodeDocumentPayload(undefined, encode({ id: "a", count: 1 })), { id: "a", count: 1 }); +test("a payload is read by its definition's formatter, a codec's, or a plain map", () => { assert.deepEqual(decodeDocumentPayload(plain, encode({ id: "a", count: 1 })), { id: "a", count: 1 }); assert.deepEqual(decodeDocumentPayload(codec, encode(["a", 1])), { id: "a", count: 1 }); assert.deepEqual(decodeDocumentPayload(formatted, new Uint8Array([0xc0])), { id: "from-formatter", count: 0 }); - // Without the definition the codec payload is its positional array, which is why a caller that holds one passes it. - assert.deepEqual(decodeDocumentPayload(undefined, encode(["a", 1])), ["a", 1]); }); test("a definition that declares both a codec and a formatter is refused where its payload is read", () => { diff --git a/packages/cultmesh-browser/test/payload-decode-owner.test.ts b/packages/cultmesh-browser/test/payload-decode-owner.test.ts deleted file mode 100644 index e5a23f78..00000000 --- a/packages/cultmesh-browser/test/payload-decode-owner.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import assert from "node:assert/strict"; -import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import test from "node:test"; - -// A document payload is decoded by cultcache-ts alone (decodeDocumentPayload); every other value on a wire goes through -// cultnet-ts's decodeWireValue / encodeWireValue. So no file of this package imports msgpack, and the manifest does not -// list it, which keeps a new direct decode from arriving unnoticed. -// Limits: a text scan. It reads the package's src files for the module specifier and the manifest for the dependency -// name. It does not follow a re-export from another package, a specifier assembled at run time, or a hand-written -// decoder; the dependency absence is what makes the specifier unresolvable here when the workspace does not hoist it. -// The package root: the nearest directory above this file (compiled or run in place) that holds a package.json. -function packageRoot(from: string): string { - return existsSync(join(from, "package.json")) ? from : packageRoot(dirname(from)); -} -const root = packageRoot(dirname(fileURLToPath(import.meta.url))); - -function sourceFiles(dir: string): string[] { - return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => - entry.isDirectory() ? sourceFiles(join(dir, entry.name)) : entry.name.endsWith(".ts") ? [join(dir, entry.name)] : []); -} - -test("no source file imports msgpack and the manifest does not list it", () => { - const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as Record | undefined>; - for (const section of ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"]) { - assert.equal(manifest[section]?.["@msgpack/msgpack"], undefined, section); - } - const files = sourceFiles(join(root, "src")); - assert.ok(files.length >= 1, "fixture: the scan reads the package's sources"); - const offenders = files.filter((file) => /@msgpack\/msgpack/.test(readFileSync(file, "utf8"))); - assert.deepEqual(offenders, []); -}); diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index f738cdcd..8dee1a39 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -127,7 +127,7 @@ export type CultMeshDocumentPublicationSource = }; export interface CultMeshPublicationDocumentBinding { - readonly schema: string | AnyCultCacheDocumentDefinition; + readonly schema: AnyCultCacheDocumentDefinition; readonly recordKey: string; readonly documentId?: string; readonly sourceId?: string; @@ -2591,54 +2591,29 @@ export function cultMeshDocumentFromCache; export function cultMeshDocumentFromStore( store: CacheBackingStore, definition: TDefinition, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, -): CultMeshDocumentHandle>; -export function cultMeshDocumentFromStore( - store: CacheBackingStore, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, options: { documentId?: string; routeHint?: CultMeshRouteHint; sourceId?: string; pollMs?: number; } = {}, -): CultMeshDocumentHandle { - const schema = typeof schemaOrDefinition === "string" - ? { schemaId: schemaOrDefinition } - : cultMeshSchemaFromDefinition(schemaOrDefinition); +): CultMeshDocumentHandle> { + const schema = cultMeshSchemaFromDefinition(definition); const resolvedSchemaId = schema.schemaId ?? schema.type; if (!resolvedSchemaId) { throw new Error("CultMesh store document requires a schema id or document type."); } - if (typeof schemaOrDefinition === "string") { - requireNonEmpty(schemaOrDefinition, "schemaId"); - } const documentId = options.documentId ?? resolvedSchemaId; - const read = async () => { + const read = async (): Promise> => { const records = await store.pullAll(); - const record = resolveCultMeshStoreDocumentRecord(records, schema, schemaOrDefinition); + const record = resolveCultMeshStoreDocumentRecord(records, schema, definition); if (!record) { throw new Error(`CultMesh store document '${documentId}' did not contain schema ${resolvedSchemaId}.`); } - return parseCultMeshDocumentValue(schemaOrDefinition, decodeCultMeshPayload(schemaOrDefinition, record.payload)); + return decodeDocumentPayload(definition, record.payload); }; return cultMeshDocument( @@ -2655,77 +2630,28 @@ export function cultMeshDocumentFromStore( ); } -export function cultMeshDocumentFromSingleFile( - path: string, - schemaId: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, -): CultMeshDocumentHandle; export function cultMeshDocumentFromSingleFile( path: string, definition: TDefinition, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, -): CultMeshDocumentHandle>; -export function cultMeshDocumentFromSingleFile( - path: string, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, options: { documentId?: string; routeHint?: CultMeshRouteHint; sourceId?: string; pollMs?: number; } = {}, -): CultMeshDocumentHandle { +): CultMeshDocumentHandle> { requireNonEmpty(path, "path"); const store = new SingleFileMessagePackBackingStore(path); - const resolvedOptions = { + return cultMeshDocumentFromStore(store, definition, { ...options, routeHint: options.routeHint ?? cultMeshRouteHint("shared-memory", path), - }; - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromStore(store, schemaOrDefinition, resolvedOptions) - : cultMeshDocumentFromStore(store, schemaOrDefinition, resolvedOptions); + }); } -export function cultMeshDocumentFromPeerSnapshot( - peer: CultNetPeer | (() => CultNetPeer | Promise), - schemaId: string, - recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, -): CultMeshDocumentHandle; export function cultMeshDocumentFromPeerSnapshot( peer: CultNetPeer | (() => CultNetPeer | Promise), definition: TDefinition, recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, -): CultMeshDocumentHandle>; -export function cultMeshDocumentFromPeerSnapshot( - peer: CultNetPeer | (() => CultNetPeer | Promise), - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, - recordKey: string, options: { documentId?: string; routeHint?: CultMeshRouteHint; @@ -2734,45 +2660,34 @@ export function cultMeshDocumentFromPeerSnapshot( pollMs?: number; messageIdPrefix?: string; } = {}, -): CultMeshDocumentHandle { - const schema = typeof schemaOrDefinition === "string" - ? { schemaId: schemaOrDefinition } - : cultMeshSchemaFromDefinition(schemaOrDefinition); +): CultMeshDocumentHandle> { + const schema = cultMeshSchemaFromDefinition(definition); const resolvedSchemaId = schema.schemaId ?? schema.type; if (!resolvedSchemaId) { throw new Error("CultMesh peer snapshot document requires a schema id or document type."); } - if (typeof schemaOrDefinition === "string") { - requireNonEmpty(schemaOrDefinition, "schemaId"); - } requireNonEmpty(recordKey, "recordKey"); const documentId = options.documentId ?? recordKey; - const read = async () => - parseCultMeshDocumentValue( - schemaOrDefinition, - decodeCultNetRawDocumentPayload( - schemaOrDefinition, - await requestCultNetRawSnapshotDocument( - peer, - resolvedSchemaId, - recordKey, - { - timeoutMs: options.timeoutMs, - messageIdPrefix: options.messageIdPrefix ?? documentId, - accepts: typeof schemaOrDefinition === "string" - ? undefined - : record => { - try { - schemaOrDefinition.schema.parse(decodeCultNetRawDocumentPayload(schemaOrDefinition, record)); - return true; - } catch { - return false; - } - }, - }, - ), - ), + const read = async (): Promise> => { + const record = await requestCultNetRawSnapshotDocument( + peer, + resolvedSchemaId, + recordKey, + { + timeoutMs: options.timeoutMs, + messageIdPrefix: options.messageIdPrefix ?? documentId, + accepts: candidate => { + try { + decodeDocumentPayload(definition, candidate.payload); + return true; + } catch { + return false; + } + }, + }, ); + return decodeDocumentPayload(definition, record.payload); + }; return cultMeshDocument( documentId, @@ -2788,36 +2703,10 @@ export function cultMeshDocumentFromPeerSnapshot( ); } -export function cultMeshDocumentFromPublication( - source: CultMeshDocumentPublicationSource, - schemaId: string, - recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, -): CultMeshDocumentHandle; export function cultMeshDocumentFromPublication( source: CultMeshDocumentPublicationSource, definition: TDefinition, recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, -): CultMeshDocumentHandle>; -export function cultMeshDocumentFromPublication( - source: CultMeshDocumentPublicationSource, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, - recordKey: string, options: { documentId?: string; routeHint?: CultMeshRouteHint; @@ -2826,7 +2715,7 @@ export function cultMeshDocumentFromPublication( pollMs?: number; messageIdPrefix?: string; } = {}, -): CultMeshDocumentHandle { +): CultMeshDocumentHandle> { if (!source) { throw new Error("CultMesh publication source is required."); } @@ -2834,14 +2723,12 @@ export function cultMeshDocumentFromPublication( switch (source.kind) { case "live-publication": { - const schema = typeof schemaOrDefinition === "string" - ? { schemaId: schemaOrDefinition } - : cultMeshSchemaFromDefinition(schemaOrDefinition); + const schema = cultMeshSchemaFromDefinition(definition); const schemaId = schema.schemaId ?? schema.type; if (!schemaId) { throw new Error("CultMesh live publication document requires a schema id or document type."); } - const parse = (value: unknown) => parseCultMeshDocumentValue(schemaOrDefinition, value); + const parse = (value: unknown) => definition.schema.parse(value) as CultCacheDocumentValue; return cultMeshDocument( options.documentId ?? recordKey, schema, @@ -2861,51 +2748,28 @@ export function cultMeshDocumentFromPublication( ); } case "single-file": - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromSingleFile(source.path, schemaOrDefinition, { - documentId: options.documentId ?? recordKey, - routeHint: options.routeHint, - sourceId: options.sourceId ?? recordKey, - pollMs: options.pollMs, - }) - : cultMeshDocumentFromSingleFile(source.path, schemaOrDefinition, { - documentId: options.documentId ?? recordKey, - routeHint: options.routeHint, - sourceId: options.sourceId ?? recordKey, - pollMs: options.pollMs, - }); + return cultMeshDocumentFromSingleFile(source.path, definition, { + documentId: options.documentId ?? recordKey, + routeHint: options.routeHint, + sourceId: options.sourceId ?? recordKey, + pollMs: options.pollMs, + }); case "store": - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromStore(source.store, schemaOrDefinition, { - documentId: options.documentId ?? recordKey, - routeHint: options.routeHint, - sourceId: options.sourceId ?? recordKey, - pollMs: options.pollMs, - }) - : cultMeshDocumentFromStore(source.store, schemaOrDefinition, { - documentId: options.documentId ?? recordKey, - routeHint: options.routeHint, - sourceId: options.sourceId ?? recordKey, - pollMs: options.pollMs, - }); + return cultMeshDocumentFromStore(source.store, definition, { + documentId: options.documentId ?? recordKey, + routeHint: options.routeHint, + sourceId: options.sourceId ?? recordKey, + pollMs: options.pollMs, + }); case "peer-snapshot": - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromPeerSnapshot(source.peer, schemaOrDefinition, recordKey, { - documentId: options.documentId ?? recordKey, - routeHint: options.routeHint ?? cultMeshRouteHint("network", source.endpoint ?? "CultNet snapshot"), - sourceId: options.sourceId ?? recordKey, - timeoutMs: options.timeoutMs, - pollMs: options.pollMs, - messageIdPrefix: options.messageIdPrefix, - }) - : cultMeshDocumentFromPeerSnapshot(source.peer, schemaOrDefinition, recordKey, { - documentId: options.documentId ?? recordKey, - routeHint: options.routeHint ?? cultMeshRouteHint("network", source.endpoint ?? "CultNet snapshot"), - sourceId: options.sourceId ?? recordKey, - timeoutMs: options.timeoutMs, - pollMs: options.pollMs, - messageIdPrefix: options.messageIdPrefix, - }); + return cultMeshDocumentFromPeerSnapshot(source.peer, definition, recordKey, { + documentId: options.documentId ?? recordKey, + routeHint: options.routeHint ?? cultMeshRouteHint("network", source.endpoint ?? "CultNet snapshot"), + sourceId: options.sourceId ?? recordKey, + timeoutMs: options.timeoutMs, + pollMs: options.pollMs, + messageIdPrefix: options.messageIdPrefix, + }); default: throw new Error(`Unsupported CultMesh publication source kind '${(source as { kind?: string }).kind}'.`); } @@ -2967,11 +2831,6 @@ export async function cultMeshSyncDocumentsFromPublication( if (!binding) { throw new Error("CultMesh publication binding is required."); } - if (typeof binding.schema === "string") { - throw new Error( - `CultMesh synced publication binding '${binding.recordKey}' requires a document definition, not only a schema id.`, - ); - } const resolvedSource = binding.source ?? (typeof source === "function" ? source(binding) : source); await node.syncDocumentFromPublication( @@ -3029,7 +2888,7 @@ export function cultMeshDocuments( } export function cultMeshPublicationDocument( - schema: string | AnyCultCacheDocumentDefinition, + schema: AnyCultCacheDocumentDefinition, recordKey: string, options: { documentId?: string; @@ -3071,33 +2930,19 @@ export function cultMeshDocumentsFromPublication( } const resolvedSource = binding.source ?? (typeof source === "function" ? source(binding) : source); - return typeof binding.schema === "string" - ? cultMeshDocumentFromPublication( - resolvedSource, - binding.schema, - binding.recordKey, - { - documentId: binding.documentId ?? binding.recordKey, - sourceId: binding.sourceId ?? binding.documentId ?? binding.recordKey, - routeHint: options.routeHint, - timeoutMs: options.timeoutMs, - pollMs: options.pollMs, - messageIdPrefix: options.messageIdPrefix, - }, - ) - : cultMeshDocumentFromPublication( - resolvedSource, - binding.schema, - binding.recordKey, - { - documentId: binding.documentId ?? binding.recordKey, - sourceId: binding.sourceId ?? binding.documentId ?? binding.recordKey, - routeHint: options.routeHint, - timeoutMs: options.timeoutMs, - pollMs: options.pollMs, - messageIdPrefix: options.messageIdPrefix, - }, - ); + return cultMeshDocumentFromPublication( + resolvedSource, + binding.schema, + binding.recordKey, + { + documentId: binding.documentId ?? binding.recordKey, + sourceId: binding.sourceId ?? binding.documentId ?? binding.recordKey, + routeHint: options.routeHint, + timeoutMs: options.timeoutMs, + pollMs: options.pollMs, + messageIdPrefix: options.messageIdPrefix, + }, + ); }), ); } @@ -4108,7 +3953,7 @@ export class CultMeshNode { messageIdPrefix: options.messageIdPrefix ?? key, accepts: record => { try { - definition.schema.parse(decodeCultNetRawDocumentPayload(definition, record)); + decodeDocumentPayload(definition, record.payload); return true; } catch { return false; @@ -5037,106 +4882,36 @@ export class CultMesh { return cultMeshDocumentFromCache(cache, definition, key, options); } - public static documentFromStore( - store: CacheBackingStore, - schemaId: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, - ): CultMeshDocumentHandle; public static documentFromStore( store: CacheBackingStore, definition: TDefinition, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, - ): CultMeshDocumentHandle>; - public static documentFromStore( - store: CacheBackingStore, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, options: { documentId?: string; routeHint?: CultMeshRouteHint; sourceId?: string; pollMs?: number; } = {}, - ): CultMeshDocumentHandle { - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromStore(store, schemaOrDefinition, options) - : cultMeshDocumentFromStore(store, schemaOrDefinition, options); + ): CultMeshDocumentHandle> { + return cultMeshDocumentFromStore(store, definition, options); } - public static documentFromSingleFile( - path: string, - schemaId: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, - ): CultMeshDocumentHandle; public static documentFromSingleFile( path: string, definition: TDefinition, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - pollMs?: number; - }, - ): CultMeshDocumentHandle>; - public static documentFromSingleFile( - path: string, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, options: { documentId?: string; routeHint?: CultMeshRouteHint; sourceId?: string; pollMs?: number; } = {}, - ): CultMeshDocumentHandle { - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromSingleFile(path, schemaOrDefinition, options) - : cultMeshDocumentFromSingleFile(path, schemaOrDefinition, options); + ): CultMeshDocumentHandle> { + return cultMeshDocumentFromSingleFile(path, definition, options); } - public static documentFromPeerSnapshot( - peer: CultNetPeer | (() => CultNetPeer | Promise), - schemaId: string, - recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, - ): CultMeshDocumentHandle; public static documentFromPeerSnapshot( peer: CultNetPeer | (() => CultNetPeer | Promise), definition: TDefinition, recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, - ): CultMeshDocumentHandle>; - public static documentFromPeerSnapshot( - peer: CultNetPeer | (() => CultNetPeer | Promise), - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, - recordKey: string, options: { documentId?: string; routeHint?: CultMeshRouteHint; @@ -5145,42 +4920,14 @@ export class CultMesh { pollMs?: number; messageIdPrefix?: string; } = {}, - ): CultMeshDocumentHandle { - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromPeerSnapshot(peer, schemaOrDefinition, recordKey, options) - : cultMeshDocumentFromPeerSnapshot(peer, schemaOrDefinition, recordKey, options); + ): CultMeshDocumentHandle> { + return cultMeshDocumentFromPeerSnapshot(peer, definition, recordKey, options); } - public static documentFromPublication( - source: CultMeshDocumentPublicationSource, - schemaId: string, - recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, - ): CultMeshDocumentHandle; public static documentFromPublication( source: CultMeshDocumentPublicationSource, definition: TDefinition, recordKey: string, - options?: { - documentId?: string; - routeHint?: CultMeshRouteHint; - sourceId?: string; - timeoutMs?: number; - pollMs?: number; - messageIdPrefix?: string; - }, - ): CultMeshDocumentHandle>; - public static documentFromPublication( - source: CultMeshDocumentPublicationSource, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, - recordKey: string, options: { documentId?: string; routeHint?: CultMeshRouteHint; @@ -5189,10 +4936,8 @@ export class CultMesh { pollMs?: number; messageIdPrefix?: string; } = {}, - ): CultMeshDocumentHandle { - return typeof schemaOrDefinition === "string" - ? cultMeshDocumentFromPublication(source, schemaOrDefinition, recordKey, options) - : cultMeshDocumentFromPublication(source, schemaOrDefinition, recordKey, options); + ): CultMeshDocumentHandle> { + return cultMeshDocumentFromPublication(source, definition, recordKey, options); } public static globalDocumentFromCache( @@ -5254,7 +4999,7 @@ export class CultMesh { } public static publicationDocument( - schema: string | AnyCultCacheDocumentDefinition, + schema: AnyCultCacheDocumentDefinition, recordKey: string, options: { documentId?: string; @@ -6331,7 +6076,7 @@ export class CultMesh { const onMessage = (message: CultNetMessage): void => { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || message.messageId !== messageId || message.document.schemaId !== receiptBinding.definition.schemaId) return; try { - const decoded = decodeCultMeshPayload(receiptBinding.definition, new Uint8Array(message.document.payload)); + const decoded = decodeDocumentPayload(receiptBinding.definition, new Uint8Array(message.document.payload)); cleanup(); resolve(receiptBinding.definition.schema.parse(decoded) as CultCacheDocumentValue); } catch (error) { cleanup(); reject(error); } @@ -6597,18 +6342,14 @@ function inferCultMeshSchemaName(schemaId: string | undefined): string | undefin function resolveCultMeshStoreDocumentRecord( records: readonly CultCacheEnvelope[], schema: CultMeshDocumentSchemaDescriptor, - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, + definition: AnyCultCacheDocumentDefinition, ): CultCacheEnvelope | undefined { const schemaId = schema.schemaId ?? schema.type; return ( records.find(candidate => schemaId !== undefined && candidate.schemaId === schemaId) ?? records.find(candidate => - cultMeshEnvelopeMatchesSchema(candidate, schema) && - (typeof schemaOrDefinition === "string" || - cultMeshEnvelopeParsesAsDefinition(candidate, schemaOrDefinition))) ?? - (typeof schemaOrDefinition === "string" - ? undefined - : records.find(candidate => cultMeshEnvelopeParsesAsDefinition(candidate, schemaOrDefinition))) + cultMeshEnvelopeMatchesSchema(candidate, schema) && cultMeshEnvelopeParsesAsDefinition(candidate, definition)) ?? + records.find(candidate => cultMeshEnvelopeParsesAsDefinition(candidate, definition)) ); } @@ -6640,7 +6381,7 @@ function cultMeshEnvelopeParsesAsDefinition( definition: AnyCultCacheDocumentDefinition, ): boolean { try { - definition.schema.parse(decodeCultMeshPayload(definition, envelope.payload)); + decodeDocumentPayload(definition, envelope.payload); return true; } catch { return false; @@ -6675,11 +6416,15 @@ function normalizeRudpDocumentPut( if (document.payloadEncoding !== "messagepack") { throw new Error(`Unsupported raw payload encoding ${document.payloadEncoding}.`); } + const definition = documents.getBySchemaId(document.schemaId)?.definition; + if (!definition) { + throw new Error("Raw document put names a schema with no registered definition."); + } return { schemaId: document.schemaId, recordKey: document.recordKey, storedAt: document.storedAt ?? new Date().toISOString(), - payload: decodeDocumentPayload(documents.getBySchemaId(document.schemaId)?.definition, document.payload), + payload: decodeDocumentPayload(definition, document.payload), sourceRuntimeId: document.sourceRuntimeId ?? null, sourceAgentId: document.sourceAgentId ?? null, sourceRole: document.sourceRole ?? null, @@ -6777,45 +6522,10 @@ async function resolveCultNetPeer( : peerOrProvider; } -// A document payload is read the way its definition's cache reads it (a codec or formatter document is not a plain -// msgpack map); a payload with no definition is plain msgpack. Both are cultcache-ts's decision. -function decodeCultMeshPayload(schemaOrDefinition: string | AnyCultCacheDocumentDefinition, payload: Uint8Array): unknown { - return decodeDocumentPayload(typeof schemaOrDefinition === "string" ? undefined : schemaOrDefinition, payload); -} - -function decodeCultNetRawDocumentPayload( - schemaOrDefinition: string | AnyCultCacheDocumentDefinition, - document: CultNetRawDocumentRecord, -): unknown { - if (document.payloadEncoding !== "messagepack") { - throw new Error(`Unsupported CultNet raw document payload encoding ${document.payloadEncoding}.`); - } - return decodeCultMeshPayload(schemaOrDefinition, toUint8Array(document.payload)); -} - -function parseCultMeshDocumentValue( - schemaOrDefinition: string | TDefinition, - value: unknown, -): unknown | CultCacheDocumentValue { - return typeof schemaOrDefinition === "string" - ? value - : schemaOrDefinition.schema.parse(value); -} - function isCultNetErrorMessage(message: CultNetMessage): message is CultNetErrorMessage { return message.schemaVersion === "cultnet.error.v0"; } -function toUint8Array(value: unknown): Uint8Array { - if (value instanceof Uint8Array) { - return value; - } - if (Array.isArray(value)) { - return Uint8Array.from(value); - } - throw new Error("CultNet raw document payload was not binary."); -} - async function bindRudpSocket(host: string, options: CultMeshRudpSocketOptions): Promise { const socket = createSocket(host.includes(":") ? "udp6" : "udp4"); const port = options.bindPort ?? 0; diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 4cdce8df..e1c4ed49 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -3018,7 +3018,12 @@ test("CultMesh TS reads remote RUDP snapshots through document handles", async ( const alias = CultMesh.documentFromPeerSnapshot( peer, - "cultmesh.note.alias.v1", + defineDocumentType({ + type: "cultmesh.note.alias", + schemaId: "cultmesh.note.alias.v1", + schema: noteDocument.schema, + name: "noteId", + }), "note:remote", { documentId: "note:remote.alias", diff --git a/packages/cultmesh-ts/test/payload-decode-owner.test.ts b/packages/cultmesh-ts/test/payload-decode-owner.test.ts deleted file mode 100644 index f6471bb4..00000000 --- a/packages/cultmesh-ts/test/payload-decode-owner.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import assert from "node:assert/strict"; -import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import test from "node:test"; - -// A document payload is decoded by cultcache-ts alone (decodeDocumentPayload); every other value on a wire goes through -// cultnet-ts's decodeWireValue / encodeWireValue. So no file of this package imports msgpack, and the manifest does not -// list it, which keeps a new direct decode from arriving unnoticed. -// Limits: a text scan. It reads the package's src files for the module specifier and the manifest for the dependency -// name. It does not follow a re-export from another package, a specifier assembled at run time, or a hand-written -// decoder; the dependency absence is what makes the specifier unresolvable here when the workspace does not hoist it. -// The package root: the nearest directory above this file (compiled or run in place) that holds a package.json. -function packageRoot(from: string): string { - return existsSync(join(from, "package.json")) ? from : packageRoot(dirname(from)); -} -const root = packageRoot(__dirname); - -function sourceFiles(dir: string): string[] { - return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => - entry.isDirectory() ? sourceFiles(join(dir, entry.name)) : entry.name.endsWith(".ts") ? [join(dir, entry.name)] : []); -} - -test("no source file imports msgpack and the manifest does not list it", () => { - const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as Record | undefined>; - for (const section of ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"]) { - assert.equal(manifest[section]?.["@msgpack/msgpack"], undefined, section); - } - const files = sourceFiles(join(root, "src")); - assert.ok(files.length >= 1, "fixture: the scan reads the package's sources"); - const offenders = files.filter((file) => /@msgpack\/msgpack/.test(readFileSync(file, "utf8"))); - assert.deepEqual(offenders, []); -}); diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index fb970906..07652497 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -5,7 +5,7 @@ import type { CultCacheDocumentValue, CultCacheEnvelope, } from "@gamecult/cultcache-ts"; -import { resolveDocumentFormatter } from "@gamecult/cultcache-ts"; +import { decodeDocumentPayload, resolveDocumentFormatter } from "@gamecult/cultcache-ts"; import { type CultNetDocumentDeleteMessage, @@ -174,7 +174,7 @@ export class CultNetDocumentRegistry { continue; } - const payload = decodeDocumentValue(binding.definition, envelope.payload); + const payload = decodeDocumentPayload(binding.definition, envelope.payload); documents.push({ schemaId, recordKey: envelope.key, @@ -343,7 +343,7 @@ export class CultNetDocumentRegistry { const payload = new Uint8Array(document.payload); for (const binding of this.#uniqueBindings()) { try { - decodeDocumentValue(binding.definition, payload); + decodeDocumentPayload(binding.definition, payload); return { binding, preserveIncomingSchemaId: false }; } catch { // Keep looking; a foreign schema id is acceptable only when the payload validates locally. @@ -429,12 +429,3 @@ function resolvePayloadSchemaVersion( - definition: TDefinition, - payload: Uint8Array, -): CultCacheDocumentValue { - const formatter = resolveDocumentFormatter(definition); - const decoded = formatter.decode(payload); - return definition.schema.parse(decoded) as CultCacheDocumentValue; -} From 0c5207b55f86aab0ad96f4683d5f6cf8cd839553 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 19:33:42 +0200 Subject: [PATCH 30/44] One typed owner of document-payload reads and writes: DocumentPayload is opaque, the definition is required DocumentPayload is an opaque object type over the same runtime Uint8Array, keyed by a string property, and types CultCacheEnvelope.payload, CultNetRawDocumentRecord.payload and CultMeshRealtimeFrame.payload. decodeDocumentPayload(definition, payload) and encodeDocumentPayload(definition, value) are the only reader and writer; documentPayloadFromBytes and documentPayloadBytes are the two named escapes at the edges where bytes enter and leave. resolveDocumentFormatter leaves the package index. The cache, cultnet replication, the mesh read paths, the provider command (now a CultCache document), and the browser's decodeCultNetPayload (definition required, cultcache-ts/document entry) all go through the owner. Compile-fail fixtures per payload-carrying package pin the type with expect-error lines. check-typescript-tests-wired.mjs fails when a test file is run by no script; cultcache-ts now runs dist-test/test/*.test.js, and the portability workflow runs the check and the cache suite. Builds in every package (tsc, src and tests); the suites run once, at the end of the pass. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/cultmesh-portability.yml | 9 +++ package-lock.json | 1 + packages/cultcache-ts/README.md | 22 +++++++ packages/cultcache-ts/package.json | 14 ++++- packages/cultcache-ts/src/cult-cache.ts | 38 +++++++----- packages/cultcache-ts/src/document.ts | 37 ++++++++++- packages/cultcache-ts/src/index.ts | 10 ++- .../single-file-messagepack-backing-store.ts | 10 ++- packages/cultcache-ts/src/types.ts | 3 +- packages/cultcache-ts/test/cult-cache.test.ts | 28 ++++----- .../test/decode-document-payload.test.ts | 17 ++++-- .../cultcache-ts/test/payload-type.test.ts | 61 +++++++++++++++++++ .../cultcache-ts/test/union-parity.test.ts | 14 ++--- packages/cultmesh-browser/README.md | 10 ++- packages/cultmesh-browser/package.json | 1 + packages/cultmesh-browser/src/index.ts | 16 ++--- .../test/cultmesh-browser.test.ts | 41 +++++++++---- .../test/payload-type.test.ts | 28 +++++++++ packages/cultmesh-ts/README.md | 12 ++-- packages/cultmesh-ts/src/index.ts | 4 +- .../src/provider-rudp-transport.ts | 5 +- .../src/provider-session-broker.ts | 4 +- .../cultmesh-ts/src/provider-session-wire.ts | 25 ++++++-- packages/cultmesh-ts/src/realtime-wire.ts | 7 ++- .../cultmesh-ts/test/codec-documents.test.ts | 38 ++++++++++++ packages/cultmesh-ts/test/cultmesh.test.ts | 21 ++++--- .../test/interop/cultmesh-quic-peer.ts | 6 +- .../cultmesh-ts/test/payload-type.test.ts | 32 ++++++++++ .../test/provider-rudp-transport.test.ts | 3 +- .../test/provider-session-wire.test.ts | 24 ++++++++ .../test/realtime-quic-consumer.test.ts | 4 +- .../test/realtime-quic-provider.test.ts | 14 +++-- .../cultmesh-ts/test/realtime-wire.test.ts | 26 ++++---- .../rudp-document-server-send-failure.test.ts | 3 +- packages/cultnet-ts/src/contracts.ts | 3 +- .../cultnet-ts/src/idunn-runtime-authority.ts | 3 +- packages/cultnet-ts/src/replication.ts | 17 +++--- .../test/idunn-runtime-authority.test.ts | 25 ++++---- .../cultnet-ts/test/replication-codec.test.ts | 14 ++--- scripts/check-typescript-tests-wired.mjs | 54 ++++++++++++++++ scripts/run-typescript-workspaces.mjs | 9 +++ scripts/test-typescript-package-closure.mjs | 12 ++++ 42 files changed, 578 insertions(+), 147 deletions(-) create mode 100644 packages/cultcache-ts/test/payload-type.test.ts create mode 100644 packages/cultmesh-browser/test/payload-type.test.ts create mode 100644 packages/cultmesh-ts/test/payload-type.test.ts create mode 100644 scripts/check-typescript-tests-wired.mjs diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index 520a4836..c64da8de 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -103,6 +103,15 @@ jobs: # Linux at all, and the browser client's never ran in CI on any OS. # packages/cultnet-ts verifies both vector files; cultmesh-browser proves # the browser client reads the shared rules rather than its own. + # No workflow ran cultcache-ts's tests, and its test script listed two files by hand while a third sat compiled + # and unrun. The wiring check fails when any TypeScript test file is run by no script, and the cache's own suite + # runs here beside the packages built on it. + - name: Check that every TypeScript test file is run by its package + run: node scripts/check-typescript-tests-wired.mjs + + - name: Verify the CultCache document owner with the TypeScript cache + run: npm run test --workspace packages/cultcache-ts + - name: Verify the CultMesh authority rules with the TypeScript verifier run: npm run test --workspace packages/cultnet-ts diff --git a/package-lock.json b/package-lock.json index d07fbc78..403865c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1202,6 +1202,7 @@ "version": "0.3.0", "license": "MIT", "dependencies": { + "@gamecult/cultcache-ts": "^0.14.0", "cultnet-ts": "^0.2.0" }, "devDependencies": { diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 1ea366f0..d37fa047 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -32,6 +32,28 @@ This is not an ORM in a fake mustache. It is for cases where you want: That keeps the polymorphic boundary tight without making callers write boilerplate by candlelight. +## Reading And Writing Payloads + +A document's bytes travel as a `DocumentPayload`: in a store envelope, a CultNet record and a +realtime frame. At run time it is the same `Uint8Array`, so nothing on the wire changes. At compile +time it is opaque: it is assignable to and from neither `Uint8Array` nor `ArrayLike`, so no +MessagePack decoder accepts it. + +- `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter, + codec or plain map and then its schema. The definition is required. +- `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter, + codec or plain map. +- `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a + writer of pre-encoded or signed bytes) and `documentPayloadBytes(payload)` is where they leave (a + store write, a frame encode, hashing or signing). Reach for them only at those edges. + +```ts +const note = decodeDocumentPayload(noteDocument, envelope.payload); // parsed Note +const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload +``` + +Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. + ## `.cc` Inspection CultCache files may use the `.cc` extension. The bytes are still the canonical diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index d31ab1b8..852c047e 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -10,12 +10,24 @@ "require": "./dist/index.js", "default": "./dist/index.js" }, + "./document": { + "types": "./dist/document.d.ts", + "require": "./dist/document.js", + "default": "./dist/document.js" + }, "./inspection": { "types": "./dist/cult-cache-inspector.d.ts", "require": "./dist/cult-cache-inspector.js", "default": "./dist/cult-cache-inspector.js" } }, + "typesVersions": { + "*": { + "document": [ + "dist/document.d.ts" + ] + } + }, "files": [ "dist" ], @@ -23,7 +35,7 @@ "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", - "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/cult-cache.test.js dist-test/test/union-parity.test.js" + "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js" }, "keywords": [ "messagepack", diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 94247d49..c0ee0b58 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,10 +1,17 @@ -import { resolveDocumentFormatter, resolveDocumentMembers } from "./document"; +import { + decodeDocumentPayload, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, + resolveDocumentFormatter, + resolveDocumentMembers, +} from "./document"; +import type { DocumentPayload } from "./document"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheDocumentAccessor, CultCacheDocumentDefinition, - CultCacheDocumentFormatter, CultCacheDocumentRegistry, CultCacheDocumentValue, CultCacheEnvelope, @@ -14,10 +21,16 @@ import type { type Accessor = (value: unknown) => string | undefined; +// A registered definition reads and writes its documents through the owner, on payloads the cache has cloned. +type RegisteredFormatter = { + encode(value: unknown): DocumentPayload; + decode(payload: DocumentPayload): unknown; +}; + type RegisteredDefinition = { readonly definition: AnyCultCacheDocumentDefinition; readonly global: boolean; - readonly formatter: CultCacheDocumentFormatter; + readonly formatter: RegisteredFormatter; readonly catalogEntry: CultCacheSchemaCatalogEntry; nameAccessor?: Accessor; indexAccessors: ReadonlyMap; @@ -674,18 +687,13 @@ export class CultCache { } } - #createFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { - const userFormatter = resolveDocumentFormatter(definition); + #createFormatter(definition: AnyCultCacheDocumentDefinition): RegisteredFormatter { + // A definition that declares both a codec and a formatter is refused here, where it is registered. + resolveDocumentFormatter(definition); return { - encode: (value: unknown) => { - const parsed = definition.schema.parse(value); - return this.#cloneBytes(userFormatter.encode(parsed)); - }, - decode: (payload: Uint8Array) => { - const decoded = userFormatter.decode(this.#cloneBytes(payload)); - return definition.schema.parse(decoded); - }, + encode: (value: unknown) => this.#cloneBytes(encodeDocumentPayload(definition, value)), + decode: (payload: DocumentPayload) => decodeDocumentPayload(definition, this.#cloneBytes(payload)), }; } @@ -1026,7 +1034,7 @@ export class CultCache { return `${type}::${indexName}`; } - #cloneBytes(payload: Uint8Array): Uint8Array { - return new Uint8Array(payload); + #cloneBytes(payload: DocumentPayload): DocumentPayload { + return documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(payload))); } } diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 50e08afb..183bd110 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -78,7 +78,7 @@ export function defineDocumentType( // The one place that decides how a document is encoded: its own formatter, or its codec's, or a plain map. A codec and a // formatter are two answers to one question, so a definition that carries both is refused wherever it enters (the -// builder resolves through here as well). +// builder resolves through here as well). Package-internal: not exported from the package index. export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { if (definition.codec && definition.formatter) { throw new Error(`CultCache document type "${definition.type}" declares both a codec and a formatter.`); @@ -91,13 +91,44 @@ const plainDocumentFormatter: CultCacheDocumentFormatter = { decode: (payload: Uint8Array) => decode(payload), }; +/** + * The bytes of one document, as they travel in a store envelope, a CultNet record or a realtime frame. + * + * At run time this is the same `Uint8Array`, so frames encode as they always did. At compile time it is opaque: it is not + * assignable to or from `Uint8Array` or `ArrayLike`, so no byte decoder accepts it. The only reader is + * `decodeDocumentPayload(definition, payload)`, which needs the definition that says how the bytes are laid out, and the + * only writer is `encodeDocumentPayload(definition, value)`. The two escapes are named: `documentPayloadFromBytes` where + * bytes enter (a validated record, a store read, tests, a writer of pre-encoded or signed bytes) and + * `documentPayloadBytes` where they leave (a store write, a frame encode, hashing or signing). + * + * The key is a plain string, not a unique symbol, so two installed copies of this package stay compatible. + */ +export interface DocumentPayload { + readonly "cultcache.documentPayload": true; + readonly byteLength: number; +} + +export function documentPayloadFromBytes(bytes: Uint8Array): DocumentPayload { + return bytes as unknown as DocumentPayload; +} + +export function documentPayloadBytes(payload: DocumentPayload): Uint8Array { + return payload as unknown as Uint8Array; +} + // The one reader of a document payload: through the definition's formatter, codec or plain map, then its schema. A // reader holds the definition or it has no business reading the bytes, so there is no definition-less form. export function decodeDocumentPayload( definition: TDefinition, - payload: Uint8Array, + payload: DocumentPayload, ): CultCacheDocumentValue { - return definition.schema.parse(resolveDocumentFormatter(definition).decode(payload)) as CultCacheDocumentValue; + const decoded = resolveDocumentFormatter(definition).decode(documentPayloadBytes(payload)); + return definition.schema.parse(decoded) as CultCacheDocumentValue; +} + +// The one writer: the definition's schema first, then its formatter, codec or plain map. +export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition, value: unknown): DocumentPayload { + return documentPayloadFromBytes(resolveDocumentFormatter(definition).encode(definition.schema.parse(value))); } // The schema members of a document: the ones it declares, else those its codec derives, else none. diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 8fae6f4c..c5dfbff1 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,15 @@ export * as codecs from "./codec"; export type { Codec, SlotsCodec } from "./codec"; export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { decodeDocumentPayload, defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document"; +export { + decodeDocumentPayload, + defineDocumentRegistry, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, +} from "./document"; +export type { DocumentPayload } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; export type { AnyCultCacheDocumentDefinition, diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index b7df1454..cdc6cf9e 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -4,6 +4,7 @@ import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; +import { documentPayloadBytes, documentPayloadFromBytes } from "./document"; import { STORE_FORMAT_VERSION, isV1Snapshot, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, @@ -62,7 +63,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { type: catalogEntry.schemaName, schemaId: record.schemaId, catalogEntry, - payload: record.payload, + payload: documentPayloadFromBytes(record.payload), storedAt: record.storedAt, }; }); @@ -82,7 +83,10 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { payload, }; }); - const parsed = envelopeArraySchema.parse(normalized) as CultCacheEnvelope[]; + const parsed: CultCacheEnvelope[] = envelopeArraySchema.parse(normalized).map((entry) => ({ + ...entry, + payload: documentPayloadFromBytes(entry.payload), + })); if (repairedLegacyPayload) { await this.#writeAll(parsed); @@ -232,7 +236,7 @@ function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] { entry.key, schemaIdForEnvelope(entry), entry.storedAt, - entry.payload, + documentPayloadBytes(entry.payload), ]); return [ diff --git a/packages/cultcache-ts/src/types.ts b/packages/cultcache-ts/src/types.ts index dada7fc0..88c20f2e 100644 --- a/packages/cultcache-ts/src/types.ts +++ b/packages/cultcache-ts/src/types.ts @@ -1,4 +1,5 @@ import type { SlotsCodec } from "./codec"; +import type { DocumentPayload } from "./document"; export type CultCacheIndexScalar = string | number | boolean | bigint; @@ -80,7 +81,7 @@ export interface CultCacheDocumentRegistry< export interface CultCacheEnvelope { key: string; type: string; - payload: Uint8Array; + payload: DocumentPayload; storedAt: string; schemaId?: string; catalogEntry?: CultCacheSchemaCatalogEntry; diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index b7c51cc1..dda2aa79 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -13,7 +13,7 @@ import { z } from "zod"; import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; -import { defineDocumentRegistry, defineDocumentType } from "../src/document"; +import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; @@ -202,7 +202,7 @@ test("CultCache fails closed on unknown or illegal persisted polymorphic state", { key: "mystery", type: "forbidden", - payload: encode({ theme: "ash" }), + payload: documentPayloadFromBytes(encode({ theme: "ash" })), storedAt: new Date().toISOString(), }, ])) @@ -219,13 +219,13 @@ test("CultCache fails closed on unknown or illegal persisted polymorphic state", { key: "one", type: "settings", - payload: encode({ theme: "ash" }), + payload: documentPayloadFromBytes(encode({ theme: "ash" })), storedAt: new Date().toISOString(), }, { key: "two", type: "settings", - payload: encode({ theme: "ember" }), + payload: documentPayloadFromBytes(encode({ theme: "ember" })), storedAt: new Date().toISOString(), }, ])) @@ -650,7 +650,7 @@ async function storeState(path: string): Promise { return []; } return (await new SingleFileMessagePackBackingStore(path).pullAll()) - .map((entry) => `${entry.type}:${entry.key}:${Buffer.from(entry.payload).toString("hex")}`) + .map((entry) => `${entry.type}:${entry.key}:${Buffer.from(documentPayloadBytes(entry.payload)).toString("hex")}`) .sort(); } @@ -821,8 +821,8 @@ test("concurrentGlobalPutsCannotBothReachDisk", async () => { const onDisk = await inner.pullAll(); assert.deepEqual(onDisk.map((entry) => entry.key), [CultCache.GLOBAL_KEY]); assert.deepEqual( - Buffer.from(onDisk[0].payload), - Buffer.from(cache.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY).payload), + Buffer.from(documentPayloadBytes(onDisk[0].payload)), + Buffer.from(documentPayloadBytes(cache.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY).payload)), ); assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { t: "second" }); const reloaded = build(new SingleFileMessagePackBackingStore(storePath)); @@ -1042,7 +1042,7 @@ test("pullWithThrowingIndexAccessorLeavesCacheUnchanged", async () => { await new SingleFileMessagePackBackingStore(storePath).push({ ...potion, key: "bomb", - payload: encode({ name: "Bomb", category: "explosive" }), + payload: documentPayloadFromBytes(encode({ name: "Bomb", category: "explosive" })), }); await assert.rejects(cache.pullAllBackingStores(), /category accessor refuses Bomb/u); @@ -1349,7 +1349,7 @@ async function writeTsInteropStore( members: interopNoteDocument.members, }, storedAt: new Date().toISOString(), - payload, + payload: documentPayloadFromBytes(payload), }); return note; } @@ -1524,7 +1524,7 @@ test("SingleFileMessagePackBackingStore refuses a dangling symlink at its path a const file = join(dir, "store.cc"); await symlink(join(volume, "store.cc"), file); const store = new SingleFileMessagePackBackingStore(file); - const envelope = { key: "k", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }; + const envelope = { key: "k", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; await assert.rejects(() => store.pullAll(), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); await assert.rejects(() => store.push(envelope), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); await assert.rejects(() => store.pushAll([envelope], { soft: true }), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); @@ -1552,7 +1552,7 @@ test("R3 decides resolve-or-refuse: a push through a live link replaces the link const volume = join(dir, "volume"); await mkdir(volume); const target = join(volume, "store.cc"); - const envelope = { key: "a", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }; + const envelope = { key: "a", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; await new SingleFileMessagePackBackingStore(target).push(envelope); const before = await readFile(target); const link = join(dir, "store.cc"); @@ -1568,7 +1568,7 @@ test("R3 decides resolve-or-refuse: a pushAll through a dangling link replaces t await mkdir(volume); const link = join(dir, "store.cc"); await symlink(join(volume, "store.cc"), link); - await new SingleFileMessagePackBackingStore(link).pushAll([{ key: "a", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }]); + await new SingleFileMessagePackBackingStore(link).pushAll([{ key: "a", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }]); assert.ok((await lstat(link)).isFile()); assert.deepEqual(await readdir(volume), []); }); @@ -1585,6 +1585,6 @@ test("CultCache inspector refuses the same vectors by name", async () => { test("a v1 store written at the base commit still reads byte for byte", async () => { const envelopes = await pullVector("v1-base.msgpack"); assert.deepEqual(envelopes.map((entry) => [entry.key, entry.type]), [["alpha", "vectors.item"], ["beta", "vectors.item"]]); - assert.deepEqual([...envelopes[0]!.payload], [0x92, 0xa5, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x01]); - assert.deepEqual([...envelopes[1]!.payload], [0x92, 0xa4, 0x62, 0x65, 0x74, 0x61, 0x02]); + assert.deepEqual([...documentPayloadBytes(envelopes[0]!.payload)], [0x92, 0xa5, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x01]); + assert.deepEqual([...documentPayloadBytes(envelopes[1]!.payload)], [0x92, 0xa4, 0x62, 0x65, 0x74, 0x61, 0x02]); }); diff --git a/packages/cultcache-ts/test/decode-document-payload.test.ts b/packages/cultcache-ts/test/decode-document-payload.test.ts index e212c54a..a18055de 100644 --- a/packages/cultcache-ts/test/decode-document-payload.test.ts +++ b/packages/cultcache-ts/test/decode-document-payload.test.ts @@ -4,8 +4,9 @@ import { test } from "node:test"; import { encode } from "@msgpack/msgpack"; import { z } from "zod"; -import { codecs, decodeDocumentPayload, defineDocumentType } from "../src"; +import { codecs, decodeDocumentPayload, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload } from "../src"; +const payloadOf = (bytes: Uint8Array) => documentPayloadFromBytes(bytes); const note = z.object({ id: z.string(), count: z.number() }); const plain = defineDocumentType({ type: "t.plain", schemaId: "t.plain.v0", schema: note, name: "id" }); const codec = defineDocumentType({ @@ -24,13 +25,19 @@ const formatted = defineDocumentType({ }); test("a payload is read by its definition's formatter, a codec's, or a plain map", () => { - assert.deepEqual(decodeDocumentPayload(plain, encode({ id: "a", count: 1 })), { id: "a", count: 1 }); - assert.deepEqual(decodeDocumentPayload(codec, encode(["a", 1])), { id: "a", count: 1 }); - assert.deepEqual(decodeDocumentPayload(formatted, new Uint8Array([0xc0])), { id: "from-formatter", count: 0 }); + assert.deepEqual(decodeDocumentPayload(plain, payloadOf(encode({ id: "a", count: 1 }))), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(codec, payloadOf(encode(["a", 1]))), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(formatted, payloadOf(new Uint8Array([0xc0]))), { id: "from-formatter", count: 0 }); }); test("a definition that declares both a codec and a formatter is refused where its payload is read", () => { const both = { ...codec, formatter: formatted.formatter } as typeof codec; - assert.throws(() => decodeDocumentPayload(both, encode(["a", 1])), (error: Error) => + assert.throws(() => decodeDocumentPayload(both, payloadOf(encode(["a", 1]))), (error: Error) => error.message === 'CultCache document type "t.codec" declares both a codec and a formatter.'); }); + +test("a payload is written by its definition's codec or formatter, the way it is read", () => { + assert.deepEqual(documentPayloadBytes(encodeDocumentPayload(codec, { id: "a", count: 1 })), encode(["a", 1])); + assert.deepEqual(documentPayloadBytes(encodeDocumentPayload(plain, { id: "a", count: 1 })), encode({ id: "a", count: 1 })); + assert.deepEqual(documentPayloadBytes(encodeDocumentPayload(formatted, { id: "a", count: 1 })), encode(["a"])); +}); diff --git a/packages/cultcache-ts/test/payload-type.test.ts b/packages/cultcache-ts/test/payload-type.test.ts new file mode 100644 index 00000000..7d7e31f7 --- /dev/null +++ b/packages/cultcache-ts/test/payload-type.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { decode, encode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { + decodeDocumentPayload, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, + type AnyCultCacheDocumentDefinition, + type CultCacheEnvelope, + type DocumentPayload, +} from "../src"; + +// The rule: a document payload is read only through its definition. The type is the enforcement, so this file is the +// proof: tsc compiles it only if every line below that carries @ts-expect-error is an error. A line that starts to +// compile makes its @ts-expect-error unused, which is itself an error, so the suite stops until the payload is opaque +// again. The function is never called; the lines are not meant to run. +export function refusedReaders(envelope: CultCacheEnvelope, definition: AnyCultCacheDocumentDefinition, payload: DocumentPayload): void { + // @ts-expect-error a plain msgpack decoder does not accept a document payload + decode(envelope.payload); + // @ts-expect-error a payload is not assignable to bytes + const bytes: Uint8Array = envelope.payload; + // @ts-expect-error nor is it a byte source a copy may be taken from + new Uint8Array(envelope.payload); + // @ts-expect-error a definition's formatter reads bytes, not a payload; the owner unwraps for it + definition.formatter!.decode(envelope.payload); + // @ts-expect-error the owner needs the definition that says how the bytes are laid out + decodeDocumentPayload(undefined, payload); + // @ts-expect-error bytes are not a payload until they enter by name + const forged: DocumentPayload = new Uint8Array(); + void [bytes, forged]; +} + +const note = defineDocumentType({ + type: "t.note", + schemaId: "t.note.v0", + schema: z.object({ id: z.string(), count: z.number() }), + name: "id", +}); + +test("a payload is the same bytes at run time, so frames and stores see no change", () => { + const bytes = encode({ id: "a", count: 1 }); + const payload = documentPayloadFromBytes(bytes); + assert.equal(documentPayloadBytes(payload), bytes); + assert.equal(payload.byteLength, bytes.byteLength); +}); + +test("the owner writes a payload through the definition's schema and reads it back through the same one", () => { + const payload = encodeDocumentPayload(note, { id: "a", count: 1 }); + assert.deepEqual(decode(documentPayloadBytes(payload)), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(note, payload), { id: "a", count: 1 }); + // A refusal names the field and never echoes the value it refused. + const refusesWithoutEchoing = (action: () => unknown) => + assert.throws(action, (error: Error) => /count/.test(error.message) && !error.message.includes("canary-value")); + refusesWithoutEchoing(() => encodeDocumentPayload(note, { id: "a", count: "canary-value" })); + refusesWithoutEchoing(() => decodeDocumentPayload(note, documentPayloadFromBytes(encode({ id: "a", count: "canary-value" })))); +}); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 5fa663b3..1502b83d 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -12,7 +12,7 @@ import { codecs } from "../src"; import { MAX_SLOT_KEY, type Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; -import { defineDocumentType } from "../src/document"; +import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; @@ -195,7 +195,7 @@ test("union-surface.cc reads through SingleFileMessagePackBackingStore and write const file = join(dir, "union-surface.cc"); await writeFile(file, await read("union-surface.cc")); const [envelope] = await new SingleFileMessagePackBackingStore(file).pullAll(); - assert.equal(hex(envelope.payload), hex(await read("union-surface.msgpack"))); + assert.equal(hex(documentPayloadBytes(envelope.payload)), hex(await read("union-surface.msgpack"))); const document = surfaceDocument(envelope.schemaId); const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); @@ -206,7 +206,7 @@ test("union-surface.cc reads through SingleFileMessagePackBackingStore and write const fresh = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(written)).build(); await fresh.put(document, envelope.key, golden); const [rewritten] = await new SingleFileMessagePackBackingStore(written).pullAll(); - assert.equal(hex(rewritten.payload), hex(envelope.payload)); + assert.equal(hex(documentPayloadBytes(rewritten.payload)), hex(documentPayloadBytes(envelope.payload))); // The reference refuses a store whose schema catalog names another type for a slot, so the catalog TS writes must be // the one C# wrote: same canonical schema text, same member names, slots and types. @@ -418,13 +418,13 @@ test("schema.parse still runs on a decoded value, and a document without a codec schema: z.object({ a: z.string().min(3) }), codec: slots<{ a: string }>({ 0: ["a", string] }), }); - const store = memoryStore([{ key: "k", type: "t", payload: encode(["xy"]), storedAt: new Date().toISOString() }]); + const store = memoryStore([{ key: "k", type: "t", payload: documentPayloadFromBytes(encode(["xy"])), storedAt: new Date().toISOString() }]); await assert.rejects(() => CultCache.builder().withDocumentType(strict).withGenericStore(store).build().pullAllBackingStores()); const plain = defineDocumentType({ type: "p", schema: z.object({ id: z.string() }) }); const holder = memoryStore([]); await CultCache.builder().withDocumentType(plain).withGenericStore(holder).build().put(plain, "k", { id: "d" }); - assert.deepEqual(decode(holder.entries[0].payload), { id: "d" }); + assert.deepEqual(decode(documentPayloadBytes(holder.entries[0].payload)), { id: "d" }); }); test("bytes and nullable round-trip through a slot record", () => { @@ -508,7 +508,7 @@ test("a definition that skips defineDocumentType still writes its codec's bytes const holder = memoryStore([]); const cache = CultCache.builder().withDocumentType(raw).withGenericStore(holder).build(); await cache.put(raw as never, "surface", golden as never); - assert.equal(hex(holder.entries[0].payload), hex(await read("union-surface.msgpack"))); + assert.equal(hex(documentPayloadBytes(holder.entries[0].payload)), hex(await read("union-surface.msgpack"))); const [snapshot] = cache.snapshot(); assert.deepEqual(snapshot.catalogEntry!.members!.map((member) => member.typeName).slice(0, 2), ["System.String", "System.Int32"]); }); @@ -687,7 +687,7 @@ test("the Int32 edges of an arm key: the largest is an arm, one past either end }); function envelopeOf(payload: Uint8Array): CultCacheEnvelope { - return { key: "surface", type: "tests.union_surface", payload, storedAt: new Date().toISOString() }; + return { key: "surface", type: "tests.union_surface", payload: documentPayloadFromBytes(payload), storedAt: new Date().toISOString() }; } function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } { diff --git a/packages/cultmesh-browser/README.md b/packages/cultmesh-browser/README.md index 59a44016..58aef6d2 100644 --- a/packages/cultmesh-browser/README.md +++ b/packages/cultmesh-browser/README.md @@ -27,9 +27,17 @@ const counter = await mesh.leaseRawDocument({ recordKey: "counter:main", }); -counter.watch(record => renderCounter(decodeCultNetPayload(record))); +counter.watch(record => renderCounter(decodeCultNetPayload(record, counterDocument))); ``` +A record's `payload` is a `DocumentPayload`, an opaque type over the same bytes, and +`decodeCultNetPayload(record, definition)` is how it is read: through the CultCache document +definition that says whether the bytes are a plain map, a codec's slot record or a +formatter's, then through the definition's schema. A plain MessagePack decode of a payload +does not compile, and `decodeCultNetPayload` takes no definition-less form. The definition +comes from `@gamecult/cultcache-ts` (this package imports only its `./document` entry, which +holds no Node built-ins). + `invoke(...)` returns the correlated provider response. Routing, schema, and envelope failures reject immediately with `CultMeshBrowserOperationError`; callers can branch on its stable `status` and `code` without parsing prose or diff --git a/packages/cultmesh-browser/package.json b/packages/cultmesh-browser/package.json index e35f143f..cd381854 100644 --- a/packages/cultmesh-browser/package.json +++ b/packages/cultmesh-browser/package.json @@ -40,6 +40,7 @@ "node": ">=20" }, "dependencies": { + "@gamecult/cultcache-ts": "^0.14.0", "cultnet-ts": "^0.2.0" }, "devDependencies": { diff --git a/packages/cultmesh-browser/src/index.ts b/packages/cultmesh-browser/src/index.ts index 2aaca7bf..5c20806c 100644 --- a/packages/cultmesh-browser/src/index.ts +++ b/packages/cultmesh-browser/src/index.ts @@ -1,3 +1,5 @@ +import { decodeDocumentPayload } from "@gamecult/cultcache-ts/document"; +import type { AnyCultCacheDocumentDefinition, CultCacheDocumentValue } from "@gamecult/cultcache-ts"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { encodeCultNetMessageForWire, @@ -761,18 +763,18 @@ export class CultMeshBrowserClient implements AsyncDisposable { } /** - * Reads a document record's payload. A document defined with a codec or a formatter is not a plain msgpack map: pass the - * formatter that reads it (a cultcache-ts `resolveDocumentFormatter(definition)`); without one the payload is read as a - * plain value, and a codec document comes back as its positional array. + * Reads a document record's payload through the definition that says how its bytes are laid out: a codec, a formatter or + * a plain map, then the definition's schema. The payload is opaque (`DocumentPayload`), so a plain msgpack decode of it + * does not compile; the definition is required. */ -export function decodeCultNetPayload( +export function decodeCultNetPayload( record: CultNetRawDocumentRecord, - formatter?: { decode(payload: Uint8Array): unknown }, -): T { + definition: TDefinition, +): CultCacheDocumentValue { if (record.payloadEncoding !== "messagepack") { throw new Error(`Unsupported CultNet document payload '${record.payloadEncoding}'.`); } - return (formatter ? formatter.decode(record.payload) : decodeWireValue(record.payload)) as T; + return decodeDocumentPayload(definition, record.payload); } function recordMatchesSchema(record: CultNetRawDocumentRecord, requestedSchema: string): boolean { diff --git a/packages/cultmesh-browser/test/cultmesh-browser.test.ts b/packages/cultmesh-browser/test/cultmesh-browser.test.ts index fb65fbf3..1206b32d 100644 --- a/packages/cultmesh-browser/test/cultmesh-browser.test.ts +++ b/packages/cultmesh-browser/test/cultmesh-browser.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { codecs, defineDocumentType, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { encodeCultNetMessageForWire, @@ -120,6 +121,13 @@ test("Odin rendezvous rejects a poisoned route without vetoing a certified fallb assert.equal(selected.endpoint, signed.route.endpoint); }); +// The counter document the fake provider serves: a plain map. +const counterDocument = defineDocumentType({ + type: "sample.counter_state", + schemaId: "sample.counter_state.v1", + schema: { parse: (value: unknown) => value as { count: number } }, +}); + test("browser client leases provider state, invokes an operation, and follows route replacement", async () => { const first = new FakeProvider("ws://127.0.0.1:4101/mesh", 4); const second = new FakeProvider("ws://127.0.0.1:4102/mesh", 9); @@ -139,11 +147,11 @@ test("browser client leases provider state, invokes an operation, and follows ro schemaId: "sample.counter_state.v1", recordKey: "counter:main", }); - assert.deepEqual(decodeCultNetPayload<{ count: number }>(lease.current!).count, 4); + assert.deepEqual(decodeCultNetPayload(lease.current!, counterDocument).count, 4); const observed: number[] = []; const unwatch = lease.watch(record => { - if (record) observed.push(decodeCultNetPayload<{ count: number }>(record).count); + if (record) observed.push(decodeCultNetPayload(record, counterDocument).count); }); const response = await client.invoke({ serviceId: "sample.counter", @@ -158,7 +166,7 @@ test("browser client leases provider state, invokes an operation, and follows ro route = second.route; await client.refreshRoute(); assert.equal(client.state, "connected"); - assert.deepEqual(decodeCultNetPayload<{ count: number }>(lease.current!).count, 9); + assert.deepEqual(decodeCultNetPayload(lease.current!, counterDocument).count, 9); assert.deepEqual(observed, [4, 6, 9]); assert.equal(first.activeSubscriptionCount, 0); assert.equal(second.activeSubscriptionCount, 1); @@ -698,7 +706,7 @@ class FakeProvider { recordKey: "counter:main", storedAt: "2026-08-17T00:00:00Z", payloadEncoding: "messagepack", - payload: encodeWireValue({ count: this.#count }), + payload: documentPayloadFromBytes(encodeWireValue({ count: this.#count })), sourceRuntimeId: this.#runtimeId, }; } @@ -976,14 +984,25 @@ function base64ToBytes(value: string): Uint8Array { return Uint8Array.from(binary, character => character.charCodeAt(0)); } -test("decodeCultNetPayload reads a payload through the formatter it is given, and plain without one", () => { - const record = { schemaId: "t.codec.v0", recordKey: "k", payloadEncoding: "messagepack", payload: encodeWireValue([7]) } as unknown as CultNetRawDocumentRecord; - const formatter = { decode: (payload: Uint8Array) => ({ count: (decodeWireValue(payload) as number[])[0] }) }; - assert.deepEqual(decodeCultNetPayload<{ count: number }>(record, formatter), { count: 7 }); - assert.deepEqual(decodeCultNetPayload(record), [7]); +test("decodeCultNetPayload reads a payload through the definition it is given: plain map, codec or formatter", () => { + const record = (bytes: Uint8Array) => ({ + schemaId: "t.v0", recordKey: "k", payloadEncoding: "messagepack", payload: documentPayloadFromBytes(bytes), + }) as unknown as CultNetRawDocumentRecord; + const schema = { parse: (value: unknown) => value as { count: number } }; + const plain = defineDocumentType({ type: "t.plain", schemaId: "t.plain.v0", schema }); + const codec = defineDocumentType({ + type: "t.codec", schemaId: "t.codec.v0", schema, codec: codecs.slots<{ count: number }>({ 0: ["count", codecs.int32] }, "T.Count"), + }); + const formatted = defineDocumentType({ + type: "t.formatted", schemaId: "t.formatted.v0", schema, + formatter: { encode: value => encodeWireValue([value.count]), decode: payload => ({ count: (decodeWireValue(payload) as number[])[0]! }) }, + }); + assert.deepEqual(decodeCultNetPayload(record(encodeWireValue({ count: 7 })), plain), { count: 7 }); + assert.deepEqual(decodeCultNetPayload(record(encodeWireValue([7])), codec), { count: 7 }); + assert.deepEqual(decodeCultNetPayload(record(encodeWireValue([7])), formatted), { count: 7 }); }); test("decodeCultNetPayload refuses a payload that is not messagepack, naming the encoding and not the payload", () => { - const record = { schemaId: "t.v0", recordKey: "k", payloadEncoding: "json", payload: encodeWireValue("secret-value") } as unknown as CultNetRawDocumentRecord; - assert.throws(() => decodeCultNetPayload(record), (error: Error) => error.message === "Unsupported CultNet document payload 'json'."); + const record = { schemaId: "t.v0", recordKey: "k", payloadEncoding: "json", payload: documentPayloadFromBytes(encodeWireValue("secret-value")) } as unknown as CultNetRawDocumentRecord; + assert.throws(() => decodeCultNetPayload(record, counterDocument), (error: Error) => error.message === "Unsupported CultNet document payload 'json'."); }); diff --git a/packages/cultmesh-browser/test/payload-type.test.ts b/packages/cultmesh-browser/test/payload-type.test.ts new file mode 100644 index 00000000..9cc95632 --- /dev/null +++ b/packages/cultmesh-browser/test/payload-type.test.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { decodeWireValue } from "cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "cultnet-ts/contracts"; + +import { decodeCultNetPayload } from "../src/index.js"; + +// The rule: a document payload is read only through its definition. A CultNet record's payload is the opaque +// DocumentPayload, so tsc compiles this file only if each line carrying an expect-error directive is an error. The function is +// never called. +export function refusedReaders(record: CultNetRawDocumentRecord): void { + // @ts-expect-error the wire value decoder is for frames and values, not for a document's payload + decodeWireValue(record.payload); + // @ts-expect-error a record's payload is not assignable to bytes + const bytes: Uint8Array = record.payload; + // @ts-expect-error the browser reader needs the definition that says how the bytes are laid out + decodeCultNetPayload(record); + // @ts-expect-error bytes are not a payload until they enter by name + const forged: DocumentPayload = new Uint8Array(); + void [bytes, forged]; +} + +test("a record's payload is the same bytes it entered with by name", () => { + const bytes = Uint8Array.from([0x90]); + assert.equal(documentPayloadBytes(documentPayloadFromBytes(bytes)), bytes); +}); diff --git a/packages/cultmesh-ts/README.md b/packages/cultmesh-ts/README.md index d3fe565b..e1fa368b 100644 --- a/packages/cultmesh-ts/README.md +++ b/packages/cultmesh-ts/README.md @@ -411,9 +411,11 @@ payload binary normalization, and MessagePack decode behind the same `CultMeshDocumentHandle` shape as local documents. The helper prefers an exact schema id match and falls back to the requested record key, so runtimes that alias the same logical document through different generated schema names can -still read the publication with one CultMesh call. Pass a CultCache document -definition instead of a raw schema id when you want a typed handle whose -`latest()` result is parsed through that definition. +still read the publication with one CultMesh call. The handle takes a CultCache +document definition, and its `latest()` result is read and parsed through that +definition: a payload is a `DocumentPayload`, which only +`decodeDocumentPayload(definition, payload)` reads, so no document is read +without the definition that says how its bytes are laid out. Use `node.syncDocumentFromPeerSnapshot(...)` or `CultMesh.syncDocumentFromPeerSnapshot(...)` when a runtime should hydrate its @@ -424,8 +426,8 @@ keep using `node.document(aliasDefinition, key)` and explicitly select `authoritativeWriter()`, `predictionWriter()`, or `observe()` without re-threading the RUDP peer. -The same typed-definition overload is available for -`CultMesh.documentFromStore(...)` and `CultMesh.documentFromSingleFile(...)`, so +`CultMesh.documentFromStore(...)` and `CultMesh.documentFromSingleFile(...)` take +the same definition, so local schema publications can be read as typed reactive documents without manually pulling records, decoding MessagePack, or parsing the payload. When the caller wants source selection to be configuration, use diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 8dee1a39..48b2f742 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -6076,9 +6076,9 @@ export class CultMesh { const onMessage = (message: CultNetMessage): void => { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || message.messageId !== messageId || message.document.schemaId !== receiptBinding.definition.schemaId) return; try { - const decoded = decodeDocumentPayload(receiptBinding.definition, new Uint8Array(message.document.payload)); + const decoded = decodeDocumentPayload(receiptBinding.definition, message.document.payload); cleanup(); - resolve(receiptBinding.definition.schema.parse(decoded) as CultCacheDocumentValue); + resolve(decoded); } catch (error) { cleanup(); reject(error); } }; const cleanup = (): void => { clearTimeout(timeout); peer.off("message", onMessage); }; diff --git a/packages/cultmesh-ts/src/provider-rudp-transport.ts b/packages/cultmesh-ts/src/provider-rudp-transport.ts index 7e94df02..fc0d75c0 100644 --- a/packages/cultmesh-ts/src/provider-rudp-transport.ts +++ b/packages/cultmesh-ts/src/provider-rudp-transport.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { createSocket, type Socket } from "node:dgram"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { encodeWireValue } from "cultnet-ts/wire"; import { CultNetPeer, @@ -171,7 +172,9 @@ class RudpProviderConnection implements CultMeshProviderConnection { recordKey: publication.recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue(publication.value), + // The one plain writer of a document payload: a publication carries no definition. The typed owner of this write is + // cultlib-gaps:follow_up:document-payload-encode-owner. + payload: documentPayloadFromBytes(encodeWireValue(publication.value)), sourceRuntimeId: this.#identity.serviceInstanceId, sourceAgentId: this.#identity.providerId, sourceRole: "provider", diff --git a/packages/cultmesh-ts/src/provider-session-broker.ts b/packages/cultmesh-ts/src/provider-session-broker.ts index 8f650e36..75240a14 100644 --- a/packages/cultmesh-ts/src/provider-session-broker.ts +++ b/packages/cultmesh-ts/src/provider-session-broker.ts @@ -1,3 +1,4 @@ +import { encodeDocumentPayload } from "@gamecult/cultcache-ts"; import { encodeWireValue } from "cultnet-ts/wire"; import type { CultNetDocumentPutRawMessage, @@ -13,6 +14,7 @@ import { cultMeshProviderSessionSchemas, decodeProviderSessionPayload, encodeProviderSessionPayload, + providerCommandDocument, type CultMeshProviderCommandReceiptWire, type CultMeshProviderCommandWire, type CultMeshProviderLeaseRenewalWire, @@ -420,7 +422,7 @@ export class CultMeshProviderSessionBroker { recordKey: `${command.providerId}/${command.serviceInstanceId}/${command.commandId}`, storedAt: this.#clock.now().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue(command), + payload: encodeDocumentPayload(providerCommandDocument, command), sourceRuntimeId: this.#options.runtimeId, sourceRole: "provider-session-broker", }, diff --git a/packages/cultmesh-ts/src/provider-session-wire.ts b/packages/cultmesh-ts/src/provider-session-wire.ts index 9bda76a6..d4884694 100644 --- a/packages/cultmesh-ts/src/provider-session-wire.ts +++ b/packages/cultmesh-ts/src/provider-session-wire.ts @@ -1,3 +1,4 @@ +import { decodeDocumentPayload, defineDocumentType } from "@gamecult/cultcache-ts"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import type { CultNetRawDocumentRecord } from "cultnet-ts"; @@ -155,6 +156,23 @@ export function assertProviderConnectEvidence( } } +// The provider command is a CultCache document like any other: written with encodeDocumentPayload and read with +// decodeDocumentPayload, so its bytes never pass through a plain decoder. +export const providerCommandDocument = defineDocumentType({ + type: cultMeshProviderSessionSchemas.command, + schemaId: cultMeshProviderSessionSchemas.command, + schema: { + parse(value: unknown): CultMeshProviderCommandWire { + const command = requireObject(value, "provider command"); + requireText(command.commandId, "commandId"); + requireText(command.commandKind, "commandKind"); + requireText(command.providerId, "providerId"); + requireText(command.serviceInstanceId, "serviceInstanceId"); + return command as unknown as CultMeshProviderCommandWire; + }, + }, +}); + export function decodeProviderCommandDocument(document: CultNetRawDocumentRecord): CultMeshProviderCommandWire { if (document.schemaId !== cultMeshProviderSessionSchemas.command) { throw new Error(`Expected ${cultMeshProviderSessionSchemas.command}, received ${document.schemaId}.`); @@ -162,12 +180,7 @@ export function decodeProviderCommandDocument(document: CultNetRawDocumentRecord if (document.payloadEncoding !== "messagepack") { throw new Error(`Provider command ${document.recordKey} must use MessagePack.`); } - const command = decodeWireValue(document.payload) as Partial; - requireText(command.commandId, "commandId"); - requireText(command.commandKind, "commandKind"); - requireText(command.providerId, "providerId"); - requireText(command.serviceInstanceId, "serviceInstanceId"); - return command as CultMeshProviderCommandWire; + return decodeDocumentPayload(providerCommandDocument, document.payload); } export function assertProviderLease(value: unknown): asserts value is CultMeshProviderLeaseWire { diff --git a/packages/cultmesh-ts/src/realtime-wire.ts b/packages/cultmesh-ts/src/realtime-wire.ts index fa647133..9fc99789 100644 --- a/packages/cultmesh-ts/src/realtime-wire.ts +++ b/packages/cultmesh-ts/src/realtime-wire.ts @@ -15,6 +15,7 @@ // the same C# test decodes. A fixture authored on one side alone would only // agree with itself. +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; import { isNullOrWhiteSpaceCSharp } from "cultnet-ts"; /** `CultMeshRealtimeWireProtocol.ApplicationProtocolName`: the QUIC ALPN. */ @@ -55,7 +56,7 @@ export interface CultMeshRealtimeFrame { producerEpoch: bigint; sequence: bigint; delivery: CultMeshRealtimeDelivery; - payload: Uint8Array; + payload: DocumentPayload; } const MAGIC = 0x31545343; @@ -123,7 +124,7 @@ export function encodeRealtimeFrame(frame: CultMeshRealtimeFrame): Uint8Array { result.set(channel, offset); offset += channel.byteLength; result.set(schema, offset); offset += schema.byteLength; result.set(body, offset); offset += body.byteLength; - result.set(frame.payload, offset); + result.set(documentPayloadBytes(frame.payload), offset); return result; } @@ -186,6 +187,6 @@ export function decodeRealtimeFrame(bytes: Uint8Array): CultMeshRealtimeFrame { // would compare equal or not depending on how it reached the decoder. // // The constructor copies and is the one type on both inputs. - payload: new Uint8Array(bytes.subarray(offset, offset + payloadLength)), + payload: documentPayloadFromBytes(new Uint8Array(bytes.subarray(offset, offset + payloadLength))), }; } diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index aeb2a14d..57be7706 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -136,3 +136,41 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r server.close(); } }); + +test("a RUDP put naming a schema the server has no definition for is refused before the handler, and the refusal does not echo the schema id", async () => { + const connectionId = 0x10203070; + const canary = "cultmesh.canary_schema_8c1f.v0"; + const stray = defineDocumentType({ + type: "cultmesh.canary_schema", + schemaId: canary, + schema: z.object({ noteId: z.string() }), + name: "noteId", + }); + const known = defineCultNetDocumentBinding({ definition: plainNote }); + const errors: Error[] = []; + const puts: unknown[] = []; + const server = CultMesh.createRudpDocumentServer("cultmesh-ts-unregistered-server", connectionId, { + documents: new CultNetDocumentRegistry([known]), + bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, + onError: (error) => errors.push(error), + onDocumentPutRaw: async (document) => { puts.push(document.payload); }, + }); + let peer: CultNetPeer | undefined; + try { + await server.start(); + peer = await CultMesh.createRudpPeer("cultmesh-ts-unregistered-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000, + }); + peer.send(new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: stray })]) + .createRawDocumentPutMessage(defineCultNetDocumentBinding({ definition: stray }), "unregistered-1", "k", { noteId: "k" })); + const startedAt = Date.now(); + while (errors.length === 0 && Date.now() - startedAt < 2_000) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(errors.length, 1); + assert.match(errors[0]!.message, /no registered definition/); + assert.ok(!errors[0]!.message.includes(canary), "the refusal names no schema id"); + assert.deepEqual(puts, [], "the handler never saw the bytes"); + } finally { + peer?.close(); + server.close(); + } +}); diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index e1c4ed49..6c972731 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -6,7 +6,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { z } from "zod"; -import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetPeer, @@ -22,6 +22,9 @@ import { } from "cultnet-ts"; import { CultMesh, CultMeshRudpUnservableDocumentError } from "../src/index"; +// A record's payload as the bytes a peer would put on the wire. +const payloadOf = (value: unknown) => documentPayloadFromBytes(encodeWireValue(value)); + const noteDocument = defineDocumentType({ type: "cultmesh.note", schemaId: "cultmesh.note.v0", @@ -268,7 +271,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa type: "runtime.generated.incompatible-note", schemaId: "runtime.generated.incompatible-note.v1", storedAt: "2026-06-27T00:00:00Z", - payload: encodeWireValue({ nope: "not the requested shape" }), + payload: payloadOf({ nope: "not the requested shape" }), catalogEntry: { schemaId: "runtime.generated.incompatible-note.v1", schemaName: "runtime.generated.incompatible-note", @@ -283,7 +286,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa type: "runtime.generated.cultmesh.note", schemaId: "runtime.generated.cultmesh.note.ui.42", storedAt: "2026-06-27T00:00:01Z", - payload: encodeWireValue({ + payload: payloadOf({ noteId: "note:foreign", body: "foreign schema id, local store shape", }), @@ -2820,7 +2823,7 @@ test("CultMesh TS RUDP document server accepts raw document puts from multiple p recordKey: "note:a", storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue("first"), + payload: payloadOf("first"), sourceRuntimeId: "client-a", }, }); @@ -2832,7 +2835,7 @@ test("CultMesh TS RUDP document server accepts raw document puts from multiple p recordKey: "note:b", storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue("second"), + payload: payloadOf("second"), sourceRuntimeId: "client-b", }, }); @@ -3316,7 +3319,7 @@ function notePut( recordKey, storedAt, payloadEncoding: "messagepack", - payload: encodeWireValue({ noteId: recordKey, body: "x".repeat(bodyLength) }), + payload: payloadOf({ noteId: recordKey, body: "x".repeat(bodyLength) }), ...(sourceRuntimeId ? { sourceRuntimeId } : {}), }, }; @@ -3350,7 +3353,7 @@ function echoReceiptBytes(put: CultNetDocumentPutRawMessage): number { recordKey: put.document.recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue(decodeWireValue(put.document.payload)), + payload: payloadOf(decodeWireValue(documentPayloadBytes(put.document.payload))), sourceRuntimeId: SERVED_BOUND_SERVER, sourceAgentId: undefined, sourceRole: "provider", @@ -3402,7 +3405,7 @@ async function withServedBoundRig( recordKey: document.recordKey, storedAt: document.storedAt, payloadEncoding: "messagepack", - payload: encodeWireValue(document.payload), + payload: payloadOf(document.payload), }, }); if (!handler.receipt) return undefined; @@ -3453,7 +3456,7 @@ function servedBodies(reply: CultNetMessage): Array<[string, number]> { assert.equal(reply.schemaVersion, "cultnet.snapshot_response_raw.v0"); return (reply as CultNetSnapshotResponseRawMessage).documents.map((document) => [ document.recordKey, - (decodeWireValue(document.payload) as { body: string }).body.length, + (decodeWireValue(documentPayloadBytes(document.payload)) as { body: string }).body.length, ]); } diff --git a/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts b/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts index aadd1265..e7e04c1a 100644 --- a/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts +++ b/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts @@ -27,6 +27,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import process from "node:process"; +import { documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { CultMeshQuicRealtimeConnector, CultMeshQuicRealtimeProvider, @@ -124,7 +126,7 @@ async function serveAsync(args: Map): Promise { producerEpoch: 1n, sequence: BigInt(sequence), delivery, - payload: Buffer.from(`frame-${sequence}`, "utf8"), + payload: documentPayloadFromBytes(Buffer.from(`frame-${sequence}`, "utf8")), }; await provider.broadcast(frame); // Printed so the harness can compare this exact frame's TypeScript @@ -215,7 +217,7 @@ async function main(): Promise { producerEpoch: frame.producerEpoch.toString(), sequence: frame.sequence.toString(), delivery: frame.delivery, - payloadHex: Buffer.from(frame.payload).toString("hex"), + payloadHex: Buffer.from(documentPayloadBytes(frame.payload)).toString("hex"), }); } finally { clearTimeout(timer); diff --git a/packages/cultmesh-ts/test/payload-type.test.ts b/packages/cultmesh-ts/test/payload-type.test.ts new file mode 100644 index 00000000..21ee2787 --- /dev/null +++ b/packages/cultmesh-ts/test/payload-type.test.ts @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { decodeWireValue } from "cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "cultnet-ts"; + +import { decodeRealtimeFrame, encodeRealtimeFrame, type CultMeshRealtimeFrame } from "../src/realtime-wire"; + +// The rule: a document payload is read only through its definition. Every payload this package touches (a CultNet +// record's, a realtime frame's) is the opaque DocumentPayload, so tsc compiles this file only if each line carrying +// an expect-error directive is an error. The function is never called. +export function refusedReaders(record: CultNetRawDocumentRecord, frame: CultMeshRealtimeFrame): void { + // @ts-expect-error the wire value decoder is for frames and values, not for a document's payload + decodeWireValue(record.payload); + // @ts-expect-error a frame's payload is not assignable to bytes + const bytes: Uint8Array = frame.payload; + // @ts-expect-error nor may a copy of it be taken as bytes + new Uint8Array(record.payload); + // @ts-expect-error bytes are not a payload until they enter by name + const forged: DocumentPayload = new Uint8Array(); + void [bytes, forged]; +} + +test("a realtime frame carries its payload as the same bytes it was given by name", () => { + const bytes = Uint8Array.from([1, 2, 3]); + const frame: CultMeshRealtimeFrame = { + channelId: "c", schemaId: "s", bodyId: "b", producerEpoch: 1n, sequence: 1n, delivery: "reliable-ordered", + payload: documentPayloadFromBytes(bytes), + }; + assert.deepEqual(documentPayloadBytes(decodeRealtimeFrame(encodeRealtimeFrame(frame)).payload), bytes); +}); diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index abb8f010..e9a9f1c5 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -1,6 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createSocket } from "node:dgram"; +import { documentPayloadBytes } from "@gamecult/cultcache-ts"; import { decodeWireValue } from "cultnet-ts/wire"; import { CultNetDocumentRegistry, @@ -46,7 +47,7 @@ test("RUDP provider transport requires broker acceptance for the full lifecycle" session.connectPayload !== undefined && decodeProviderConnectEvidence(session.connectPayload).sessionToken === "signed-provider-token", acceptPublication: (_identity, publicationId, document) => { - accepted.set(publicationId, decodeWireValue(document.payload)); + accepted.set(publicationId, decodeWireValue(documentPayloadBytes(document.payload))); acceptedProvenance.set(publicationId, { runtime: document.sourceRuntimeId, agent: document.sourceAgentId, diff --git a/packages/cultmesh-ts/test/provider-session-wire.test.ts b/packages/cultmesh-ts/test/provider-session-wire.test.ts index ac607163..a9fd3345 100644 --- a/packages/cultmesh-ts/test/provider-session-wire.test.ts +++ b/packages/cultmesh-ts/test/provider-session-wire.test.ts @@ -1,10 +1,16 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { encodeWireValue } from "cultnet-ts/wire"; + import { + cultMeshProviderSessionSchemas, + decodeProviderCommandDocument, decodeProviderConnectEvidence, encodeProviderConnectEvidence, encodeProviderSessionPayload, + providerCommandDocument, } from "../src/provider-session-wire"; const REGISTRATION_GOLDEN = "hqpwcm92aWRlcklkqGFldGhlcmlhsXNlcnZpY2VJbnN0YW5jZUlkq2FldGhlcmlhLTQyqmVuZHBvaW50SWSvYWV0aGVyaWEtcHVibGljp3ZlcnNlSWSmcHVibGljuHJlcXVlc3RlZExlYXNlRHVyYXRpb25Nc811MLBhdXRob3JpdHlMZWFzZUlkq2F1dGhvcml0eS03"; @@ -41,3 +47,21 @@ test("provider Connect evidence keeps transport generation separate from authori TOKENLESS_CONNECT_EVIDENCE_GOLDEN, ); }); + +test("a provider command is a CultCache document: written and read by its definition, and a refusal names no value", () => { + const command = { commandId: "c1", commandKind: "k", providerId: "p", serviceInstanceId: "s", payload: { n: 1 } }; + const payload = encodeDocumentPayload(providerCommandDocument, command); + assert.deepEqual(documentPayloadBytes(payload), encodeWireValue(command), "the wire bytes are the plain map they always were"); + const record = (bytes: DocumentPayload) => ({ + schemaId: cultMeshProviderSessionSchemas.command, recordKey: "p/s/c1", storedAt: "2026-10-11T00:00:00Z", + payloadEncoding: "messagepack" as const, payload: bytes, + }); + assert.deepEqual(decodeProviderCommandDocument(record(payload)), command); + + const refused = documentPayloadFromBytes(encodeWireValue({ ...command, providerId: "canary-provider-5d2e", commandKind: "" })); + assert.throws( + () => decodeProviderCommandDocument(record(refused)), + (error: Error) => /commandKind/.test(error.message) && !error.message.includes("canary-provider-5d2e"), + ); + assert.throws(() => encodeDocumentPayload(providerCommandDocument, { ...command, commandId: " " }), /commandId/); +}); diff --git a/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts b/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts index 8a2063fe..acf06a41 100644 --- a/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts +++ b/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts @@ -14,6 +14,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import test from "node:test"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { canonicalRoute, type CultMeshAuthorityRouteView, @@ -154,7 +156,7 @@ function testFrame(overrides: Partial = {}): CultMeshReal producerEpoch: 7n, sequence: 1n, delivery: "latest-only", - payload: new Uint8Array([3, 4, 5]), + payload: documentPayloadFromBytes(new Uint8Array([3, 4, 5])), ...overrides, }; } diff --git a/packages/cultmesh-ts/test/realtime-quic-provider.test.ts b/packages/cultmesh-ts/test/realtime-quic-provider.test.ts index 8a541641..d174609b 100644 --- a/packages/cultmesh-ts/test/realtime-quic-provider.test.ts +++ b/packages/cultmesh-ts/test/realtime-quic-provider.test.ts @@ -13,6 +13,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import test from "node:test"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { CultMeshQuicNativeRuntime, } from "../src/realtime-quic-native"; @@ -63,7 +65,7 @@ function testFrame(overrides: Partial = {}): CultMeshReal producerEpoch: 7n, sequence: 1n, delivery: "latest-only", - payload: new Uint8Array([3, 4, 5]), + payload: documentPayloadFromBytes(new Uint8Array([3, 4, 5])), ...overrides, }; } @@ -613,8 +615,8 @@ test("a peer attaching after two latest-only broadcasts for one key receives exa if (!nativeBridgeAvailable()) return void t.skip("no native bridge available"); const provider = await startProvider(); try { - await provider.broadcast(testFrame({ sequence: 1n, payload: new Uint8Array([1]) })); - await provider.broadcast(testFrame({ sequence: 2n, payload: new Uint8Array([2]) })); + await provider.broadcast(testFrame({ sequence: 1n, payload: documentPayloadFromBytes(new Uint8Array([1])) })); + await provider.broadcast(testFrame({ sequence: 2n, payload: documentPayloadFromBytes(new Uint8Array([2])) })); // Attach after both broadcasts have already resolved: nothing in this // peer's outbox came from `broadcast()`'s own fan-out, only from the @@ -636,11 +638,11 @@ test("retention keeps the newer generation when an older one broadcasts afterwar if (!nativeBridgeAvailable()) return void t.skip("no native bridge available"); const provider = await startProvider(); try { - await provider.broadcast(testFrame({ sequence: 5n, payload: new Uint8Array([5]) })); + await provider.broadcast(testFrame({ sequence: 5n, payload: documentPayloadFromBytes(new Uint8Array([5])) })); // An older generation broadcast after a newer one must not overwrite the // retained frame: mutating the retention comparison to a plain // last-write-wins assignment would regress this. - await provider.broadcast(testFrame({ sequence: 3n, payload: new Uint8Array([3]) })); + await provider.broadcast(testFrame({ sequence: 3n, payload: documentPayloadFromBytes(new Uint8Array([3])) })); const consumer = await dialProvider(provider); try { @@ -672,7 +674,7 @@ test("a reliable-ordered frame is never retained and is not delivered to a late // after a real latest-only frame, so the assertion below proves the // reliable-ordered frame specifically was never retained, rather than // merely proving the provider had nothing at all to seed. - await provider.broadcast(testFrame({ delivery: "latest-only", sequence: 1n, payload: new Uint8Array([1]) })); + await provider.broadcast(testFrame({ delivery: "latest-only", sequence: 1n, payload: documentPayloadFromBytes(new Uint8Array([1])) })); const secondConsumer = await dialProvider(provider); try { const seeded = await secondConsumer.receiveFrame(); diff --git a/packages/cultmesh-ts/test/realtime-wire.test.ts b/packages/cultmesh-ts/test/realtime-wire.test.ts index 93f23365..d4497b84 100644 --- a/packages/cultmesh-ts/test/realtime-wire.test.ts +++ b/packages/cultmesh-ts/test/realtime-wire.test.ts @@ -16,6 +16,8 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { join } from "node:path"; +import { documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { CULTMESH_REALTIME_ALPN, CULTMESH_REALTIME_CONNECTION_CLOSE_CODE, @@ -89,7 +91,7 @@ function frameOf(fields: VectorFields): CultMeshRealtimeFrame { producerEpoch: 0n, sequence: 0n, delivery: fields.delivery, - payload: payload(fields.payload), + payload: documentPayloadFromBytes(payload(fields.payload)), }; } @@ -175,7 +177,7 @@ const sample = (overrides: Partial = {}): CultMeshRealtim producerEpoch: 1n, sequence: 2n, delivery: "reliable-ordered", - payload: Uint8Array.from([1, 2, 3]), + payload: documentPayloadFromBytes(Uint8Array.from([1, 2, 3])), ...overrides, }); @@ -199,7 +201,7 @@ test("the fixed header is 37 little-endian bytes in the reference's order", () = const encoded = encodeRealtimeFrame(sample({ channelId: "ch", schemaId: "sch", bodyId: "bdy", producerEpoch: 0x0102030405060708n, sequence: 0x1112131415161718n, - delivery: "unreliable", payload: Uint8Array.from([0xaa]), + delivery: "unreliable", payload: documentPayloadFromBytes(Uint8Array.from([0xaa])), })); const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength); assert.equal(view.getUint32(0, true), 0x31545343, "magic, little-endian"); @@ -258,11 +260,11 @@ test("an identity of exactly 65535 UTF-8 bytes encodes and one byte more is refu }); test("a payload at the 64 MiB ceiling encodes and one byte more is refused", () => { - const encoded = encodeRealtimeFrame(sample({ payload: ramp(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES) })); + const encoded = encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(ramp(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES)) })); assert.equal(encoded.byteLength, 37 + 15 + 20 + 6 + CULTMESH_REALTIME_MAX_PAYLOAD_BYTES); assert.equal(decodeRealtimeFrame(encoded).payload.byteLength, CULTMESH_REALTIME_MAX_PAYLOAD_BYTES); assert.throws( - () => encodeRealtimeFrame(sample({ payload: new Uint8Array(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES + 1) })), + () => encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(new Uint8Array(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES + 1)) })), /payload exceeds the QUIC wire limit/, ); }); @@ -363,16 +365,16 @@ test("decode refuses a negative payload length that the length sum cancels out", }); test("decode reads exactly the viewed bytes of an offset view, not the buffer behind it", () => { - const good = encodeRealtimeFrame(sample({ payload: Uint8Array.from([9, 8, 7]) })); + const good = encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(Uint8Array.from([9, 8, 7])) })); const pool = new Uint8Array(good.byteLength + 64).fill(0xa5); pool.set(good, 17); const view = pool.subarray(17, 17 + good.byteLength); - assert.deepEqual(decodeRealtimeFrame(view).payload, Uint8Array.from([9, 8, 7])); + assert.deepEqual(documentPayloadBytes(decodeRealtimeFrame(view).payload), Uint8Array.from([9, 8, 7])); assert.equal(decodeRealtimeFrame(view).channelId, "aetheria.zone-1"); // And the decoded payload does not alias the frame it came from. const decoded = decodeRealtimeFrame(good); good[good.byteLength - 1] = 0; - assert.deepEqual(decoded.payload, Uint8Array.from([9, 8, 7])); + assert.deepEqual(documentPayloadBytes(decoded.payload), Uint8Array.from([9, 8, 7])); }); // The `Uint8Array` case above passes with a plain `bytes.slice(...)`; this one @@ -380,12 +382,12 @@ test("decode reads exactly the viewed bytes of an offset view, not the buffer be // `Buffer` is what a socket read hands a Node caller, so the aliasing bug this // pins is the one that would actually reach a consumer. test("the decoded payload copies out of a Node Buffer too, which overrides slice", () => { - const frame = Buffer.from(encodeRealtimeFrame(sample({ payload: Uint8Array.from([9, 8, 7]) }))); + const frame = Buffer.from(encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(Uint8Array.from([9, 8, 7])) }))); const decoded = decodeRealtimeFrame(frame); frame[frame.byteLength - 1] = 0; - assert.deepEqual(decoded.payload, Uint8Array.from([9, 8, 7]), "payload unchanged by a later write to the frame"); - assert.equal(Buffer.isBuffer(decoded.payload), false, "the copy is a plain Uint8Array, not a Buffer"); - assert.notEqual(decoded.payload.buffer, frame.buffer, "the copy has its own backing store"); + assert.deepEqual(documentPayloadBytes(decoded.payload), Uint8Array.from([9, 8, 7]), "payload unchanged by a later write to the frame"); + assert.equal(Buffer.isBuffer(documentPayloadBytes(decoded.payload)), false, "the copy is a plain Uint8Array, not a Buffer"); + assert.notEqual(documentPayloadBytes(decoded.payload).buffer, frame.buffer, "the copy has its own backing store"); // The three identities need no such check: `TextDecoder` returns strings, which // hold no reference to the bytes they were decoded from. assert.equal(decoded.channelId, "aetheria.zone-1"); diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts index 8d0425e4..713a3bee 100644 --- a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -5,6 +5,7 @@ import assert from "node:assert/strict"; import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; import test from "node:test"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { encodeWireValue } from "cultnet-ts/wire"; import { CultNetDocumentRegistry, @@ -43,7 +44,7 @@ function put(recordKey: string) { recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack" as const, - payload: encodeWireValue(recordKey), + payload: documentPayloadFromBytes(encodeWireValue(recordKey)), }, }; } diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 4bfdafad..31188790 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -1,3 +1,4 @@ +import type { DocumentPayload } from "@gamecult/cultcache-ts/document"; import Ajv2020, { type ValidateFunction } from "ajv/dist/2020.js"; import documentRecordSchema from "../contracts/cultnet/cultnet.document-record.schema.json"; @@ -200,7 +201,7 @@ export interface CultNetRawDocumentRecord { recordKey: string; storedAt: string; payloadEncoding: CultNetRawPayloadEncoding; - payload: Uint8Array; + payload: DocumentPayload; sourceRuntimeId?: string; sourceAgentId?: string; sourceRole?: string; diff --git a/packages/cultnet-ts/src/idunn-runtime-authority.ts b/packages/cultnet-ts/src/idunn-runtime-authority.ts index 7f7b882f..2ec5456c 100644 --- a/packages/cultnet-ts/src/idunn-runtime-authority.ts +++ b/packages/cultnet-ts/src/idunn-runtime-authority.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts/document"; import { decode, encode } from "@msgpack/msgpack"; import { @@ -278,7 +279,7 @@ export function createIdunnRuntimeSigner(options: IdunnRuntimeSignerOptions): Id recordKey: expected.target, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: new Uint8Array(signed.payload), + payload: documentPayloadFromBytes(new Uint8Array(signed.payload)), sourceRuntimeId: expected.runtimeId, sourceRole: "runtime-presence-health-publisher", tags: ["cultnet.transport.rudp.v0", "runtime-presence"], diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index 07652497..e12eaf8e 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -5,7 +5,12 @@ import type { CultCacheDocumentValue, CultCacheEnvelope, } from "@gamecult/cultcache-ts"; -import { decodeDocumentPayload, resolveDocumentFormatter } from "@gamecult/cultcache-ts"; +import { + decodeDocumentPayload, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, +} from "@gamecult/cultcache-ts"; import { type CultNetDocumentDeleteMessage, @@ -134,8 +139,6 @@ export class CultNetDocumentRegistry { tags?: string[]; } = {}, ): CultNetDocumentPutRawMessage { - const parsed = binding.definition.schema.parse(value); - const formatter = resolveDocumentFormatter(binding.definition); return { schemaVersion: "cultnet.document_put_raw.v0", messageId, @@ -144,7 +147,7 @@ export class CultNetDocumentRegistry { recordKey: documentKey, storedAt: options.storedAt ?? new Date().toISOString(), payloadEncoding: "messagepack", - payload: formatter.encode(parsed), + payload: encodeDocumentPayload(binding.definition, value), sourceRuntimeId: options.sourceRuntimeId, sourceAgentId: options.sourceAgentId, sourceRole: options.sourceRole, @@ -281,7 +284,7 @@ export class CultNetDocumentRegistry { schemaId: resolution.preserveIncomingSchemaId ? message.document.schemaId : schemaIdForBinding(binding), - payload: new Uint8Array(message.document.payload), + payload: documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(message.document.payload))), storedAt: message.document.storedAt, }, }; @@ -340,7 +343,7 @@ export class CultNetDocumentRegistry { return { binding: exact, preserveIncomingSchemaId: true }; } - const payload = new Uint8Array(document.payload); + const payload = documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(document.payload))); for (const binding of this.#uniqueBindings()) { try { decodeDocumentPayload(binding.definition, payload); @@ -367,7 +370,7 @@ export class CultNetDocumentRegistry { recordKey: envelope.key, storedAt: envelope.storedAt, payloadEncoding: "messagepack", - payload: new Uint8Array(envelope.payload), + payload: documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(envelope.payload))), }; } } diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index b5c9236a..2aff6d92 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -4,6 +4,7 @@ import { spawnSync } from "node:child_process"; import dgram from "node:dgram"; import fs from "node:fs"; import path from "node:path"; +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; import { decode, encode } from "@msgpack/msgpack"; import { @@ -133,18 +134,18 @@ test("rejects a credential file holding more than one envelope", (context) => { ); }); -function presenceSlot(document: { payload: Uint8Array }, slot: number): unknown { - return (decode(document.payload) as unknown[])[slot]; +function presenceSlot(document: { payload: DocumentPayload }, slot: number): unknown { + return (decode(documentPayloadBytes(document.payload)) as unknown[])[slot]; } // Every committed vector is asserted, dead ones included. Stateless sets sign at sequence 1. for (const set of ["web", "service", "rudp-route"] as const) { test(`${set}: a fresh signer answers warming, and after reportHealth('active') answers active, byte for byte as the Rust-verified vectors`, (context) => { freezeClock(context); - assert.deepEqual(Buffer.from(signerFor(context, set).answerRouteObservation(CHALLENGE).payload), fixtureFile(set, "presence-warming.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(signerFor(context, set).answerRouteObservation(CHALLENGE).payload)), fixtureFile(set, "presence-warming.bin")); const active = signerFor(context, set); active.reportHealth("active"); - assert.deepEqual(Buffer.from(active.answerRouteObservation(CHALLENGE).payload), fixtureFile(set, "presence-active.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(active.answerRouteObservation(CHALLENGE).payload)), fixtureFile(set, "presence-active.bin")); }); } @@ -152,7 +153,7 @@ test("a shortfall signs the vector Rust correlates to the typed capacity disagre freezeClock(context); const signer = signerFor(context, "service", -1); signer.reportHealth("active"); - assert.deepEqual(Buffer.from(signer.answerRouteObservation(CHALLENGE).payload), fixtureFile("service", "presence-active-below-minimum.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(signer.answerRouteObservation(CHALLENGE).payload)), fixtureFile("service", "presence-active-below-minimum.bin")); }); test("the signer owns the reported health: a fresh signer is warming, reportHealth moves it, the sequence keeps counting", (context) => { @@ -197,7 +198,7 @@ async function startOdinPeer(context: TestContext, endpoint: string) { context.after(() => peer.close()); const state = { reject: false, - received: [] as Array<{ schemaVersion?: string; document: { payload: Uint8Array } }>, + received: [] as Array<{ schemaVersion?: string; document: { payload: DocumentPayload } }>, responseAck: undefined as number | undefined, failure: undefined as Error | undefined, }; @@ -239,7 +240,7 @@ test("the Odin publisher publishes the signer's document to the endpoint Expecte await publisher.publish("accepted presence"); assert.equal(peer.received[0]?.schemaVersion, "cultnet.document_put_raw.v0"); - const published = decode(peer.received[0]!.document.payload) as unknown[]; + const published = decode(documentPayloadBytes(peer.received[0]!.document.payload)) as unknown[]; assert.equal(published[RUNTIME_PRESENCE_SLOT.state], "warming"); assert.equal(published[RUNTIME_PRESENCE_SLOT.publisherSequence], 1); assert.equal(peer.responseAck, 2, "the response data packet must acknowledge the publisher's reliable schema packet"); @@ -257,7 +258,7 @@ test("the Odin publisher leaves a capability shortfall to the Rust correlation i signer.reportHealth("active"); const publisher = createIdunnRuntimePresencePublisher({ signer, endpoint }); await publisher.publish("below minimum"); - const capabilities = (decode(peer.received[0]!.document.payload) as unknown[])[RUNTIME_PRESENCE_SLOT.capabilities] as unknown[][]; + const capabilities = (decode(documentPayloadBytes(peer.received[0]!.document.payload)) as unknown[])[RUNTIME_PRESENCE_SLOT.capabilities] as unknown[][]; assert.equal(capabilities[0]![3], expectedFacts("service").capabilities[0]!.capacity - 1); }); @@ -304,16 +305,16 @@ function statefulSigner(context: TestContext, lease = "lease.cc") { test("stateful: warming, active and degraded match the Rust-verified vectors byte for byte, active and degraded bound to Rust's lease", (context) => { freezeClock(context); const signer = statefulSigner(context); - assert.deepEqual(Buffer.from(signer.answerRouteObservation(CHALLENGE).payload), fixtureFile(STATEFUL, "presence-warming.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(signer.answerRouteObservation(CHALLENGE).payload)), fixtureFile(STATEFUL, "presence-warming.bin")); signer.reportHealth("active"); const active = signer.answerRouteObservation(CHALLENGE); - assert.deepEqual(Buffer.from(active.payload), fixtureFile(STATEFUL, "presence-active.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(active.payload)), fixtureFile(STATEFUL, "presence-active.bin")); assert.match(String(presenceSlot(active, RUNTIME_PRESENCE_SLOT.writeLeaseSha256)), /^sha256-[0-9a-f]{64}$/); const degraded = statefulSigner(context); degraded.answerRouteObservation(CHALLENGE); degraded.reportHealth("degraded"); - assert.deepEqual(Buffer.from(degraded.answerRouteObservation(CHALLENGE).payload), fixtureFile(STATEFUL, "presence-degraded.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(degraded.answerRouteObservation(CHALLENGE).payload)), fixtureFile(STATEFUL, "presence-degraded.bin")); }); test("a warming answered on the route becomes a lease target: Rust's lease naming it is accepted", (context) => { @@ -321,7 +322,7 @@ test("a warming answered on the route becomes a lease target: Rust's lease namin const signer = statefulSigner(context); assert.throws(() => signer.assertWriteLease(), /does not match the current Expected incarnation/, "no warming signed yet"); signer.answerRouteObservation(CHALLENGE); - assert.equal(signer.assertWriteLease(), presenceSlot({ payload: fixtureFile(STATEFUL, "presence-active.bin") }, RUNTIME_PRESENCE_SLOT.writeLeaseSha256)); + assert.equal(signer.assertWriteLease(), presenceSlot({ payload: documentPayloadFromBytes(fixtureFile(STATEFUL, "presence-active.bin")) }, RUNTIME_PRESENCE_SLOT.writeLeaseSha256)); }); for (const [lease, reason] of [["lease-other-warming.cc", "names another warming digest"], ["lease-other-target.cc", "names another target"]] as const) { diff --git a/packages/cultnet-ts/test/replication-codec.test.ts b/packages/cultnet-ts/test/replication-codec.test.ts index 0c6cd632..a93622ed 100644 --- a/packages/cultnet-ts/test/replication-codec.test.ts +++ b/packages/cultnet-ts/test/replication-codec.test.ts @@ -5,7 +5,7 @@ import { join, resolve } from "node:path"; import test from "node:test"; import { z } from "zod"; -import { CultCache, codecs, defineDocumentType, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; +import { CultCache, codecs, defineDocumentType, documentPayloadBytes, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, defineCultNetDocumentBinding } from "../src"; @@ -55,23 +55,23 @@ test("a codec document replicates as the bytes the C# reference writes, and a sn const target = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "target.cc"))).build(); const put = documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-1", "n1", value); - assert.equal(hex(put.document.payload), hex(reference), "a raw put carries the reference's record, not a map"); + assert.equal(hex(documentPayloadBytes(put.document.payload)), hex(reference), "a raw put carries the reference's record, not a map"); await documents.applyRawDocumentPutMessage(origin, put); - assert.equal(hex(origin.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(origin.getRequiredEnvelope(mixedDefinition, "n1").payload)), hex(reference)); const raw = documents.createRawSnapshotResponse(origin, "snapshot-raw"); - assert.equal(hex(raw.documents[0].payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(raw.documents[0].payload)), hex(reference)); await documents.applyRawSnapshotResponse(target, raw); assert.deepEqual(target.getRequired(mixedDefinition, "n1"), value); - assert.equal(hex(target.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(target.getRequiredEnvelope(mixedDefinition, "n1").payload)), hex(reference)); // The value-level snapshot decodes the stored record through the codec, so a peer applying it parses a record. const values = documents.createSnapshotResponse(origin, "snapshot-values"); assert.deepEqual(values.documents[0].payload, value); const other = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "other.cc"))).build(); await documents.applySnapshotResponse(other, values); - assert.equal(hex(other.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(other.getRequiredEnvelope(mixedDefinition, "n1").payload)), hex(reference)); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -100,5 +100,5 @@ test("a document with neither a codec nor a formatter replicates as a map, as be }); const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: plain })]); const put = documents.createRawDocumentPutMessage(documents.get("tests.plain_record")!, "put-3", "n1", { id: "n1" }); - assert.equal(hex(put.document.payload), "81a26964a26e31"); + assert.equal(hex(documentPayloadBytes(put.document.payload)), "81a26964a26e31"); }); diff --git a/scripts/check-typescript-tests-wired.mjs b/scripts/check-typescript-tests-wired.mjs new file mode 100644 index 00000000..8395699d --- /dev/null +++ b/scripts/check-typescript-tests-wired.mjs @@ -0,0 +1,54 @@ +// Fails when a TypeScript test file exists that no npm script of its package runs. +// +// A package's `test` script names the compiled files it runs, either by glob or one by one, and a file that is compiled +// and listed nowhere passes every suite while proving nothing: cultcache-ts's decode-document-payload test sat in that +// state behind an explicit two-file list. Every test/**/*.test.ts maps to its dist-test/**/*.test.js, and that path has +// to match an argument of `node --test` in the package's `test` script, or in `test:interop` for test/interop. +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const rootManifest = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")); + +function testFiles(dir, prefix = "") { + return readdirSync(dir, { withFileTypes: true }).flatMap(entry => + entry.isDirectory() + ? testFiles(join(dir, entry.name), `${prefix}${entry.name}/`) + : entry.name.endsWith(".test.ts") ? [`${prefix}${entry.name}`] : []); +} + +// The paths `node --test` is given in a script: the words of its command after `--test` that are not flags. +function nodeTestArguments(script) { + const words = script?.split("&&").map(part => part.trim().split(/\s+/)).find(part => part[0] === "node" && part.includes("--test")); + return words ? words.slice(words.indexOf("--test") + 1).filter(argument => !argument.startsWith("--")) : []; +} + +function matches(pattern, path) { + const [directory, file] = [dirname(pattern), pattern.slice(dirname(pattern).length + 1)]; + if (directory !== dirname(path)) return false; + return new RegExp(`^${file.split("*").map(part => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*")}$`) + .test(path.slice(directory.length + 1)); +} + +const unwired = []; +let checked = 0; +for (const workspace of rootManifest.workspaces) { + const packageRoot = join(repoRoot, workspace); + const scripts = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")).scripts ?? {}; + for (const file of testFiles(join(packageRoot, "test"))) { + checked += 1; + const compiled = `dist-test/test/${file.replace(/\.ts$/, ".js")}`; + const scriptName = file.startsWith("interop/") ? "test:interop" : "test"; + if (!nodeTestArguments(scripts[scriptName]).some(pattern => matches(pattern, compiled))) { + unwired.push(`${workspace}: test/${file} is not run by its package's ${scriptName} script`); + } + } +} + +if (checked === 0) throw new Error("The wiring check found no test files; it is looking in the wrong place."); +if (unwired.length > 0) { + console.error(unwired.join("\n")); + process.exit(1); +} +console.log(`${checked} TypeScript test files, all run by their package's script.`); diff --git a/scripts/run-typescript-workspaces.mjs b/scripts/run-typescript-workspaces.mjs index da881464..0ef34005 100644 --- a/scripts/run-typescript-workspaces.mjs +++ b/scripts/run-typescript-workspaces.mjs @@ -1,5 +1,6 @@ import { execFileSync } from "node:child_process"; import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; const command = process.argv[2]; if (!command) throw new Error("Usage: node scripts/run-typescript-workspaces.mjs