diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 06e1f3f8..1ea366f0 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -272,13 +272,13 @@ const sceneDocument = defineDocumentType({ - The leaves are `string`, `boolean`, `bytes`, `int32`, `int64`, `float32` and `float64`, each writing the MessagePack the C# type writes and refusing what C# refuses (a float in an int slot, an `int` outside Int32). `int64` reads a number when the value is a safe integer and a bigint beyond 2^53, and refuses a number beyond 2^53 on write. `list`, `array` - and `nullable` wrap a leaf, a union or a record. + and `nullable` wrap a leaf, a union or a record. A `float32` slot rounds an integer once, as the reference's cast does. - Every codec carries the reference's type name, and a store's schema catalog is derived from those names, because the reference refuses a store whose slot names another type. The leaves know theirs; give a union or a record its C# name as the last argument of `union(name, arms, typeName)` or `slots(spec, typeName)`. A codec with a slot it cannot name refuses to derive members: pass `members` instead. -- An arm key is an integer within Int32, in any MessagePack integer encoding. A float key (even `1.0`) is refused on - read, and a union refuses a key outside Int32 when it is defined. +- An arm key is an integer within Int32 (negative too), in any MessagePack integer encoding. A float key (even `1.0`) is + refused on read, and a union refuses a key outside Int32 when it is defined. A slot key is within 0 to Int32 max. - `codec` supplies the document's formatter and, unless `members` is given, its catalog members. A definition may not carry both `codec` and `formatter`. `schema.parse` still runs on every decoded value. diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 508fb9d6..c7d2736d 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -87,6 +87,19 @@ export const int32 = integerCodec("System.Int32", "an int32", -(2n ** 31n), 2n * // C# long. A number when the value is a safe integer, a bigint beyond 2^53 (so 64-bit values never lose precision). export const int64 = integerCodec("System.Int64", "an int64", -(2n ** 63n), 2n ** 63n - 1n); +// A 64-bit integer to the nearest float32 in one rounding, ties to even, as the reference's cast does. Number(value) +// first would round to a double and then again, and the two roundings can disagree. +function bigintToFloat32(value: bigint): number { + const magnitude = value < 0n ? -value : value; + const excess = magnitude.toString(2).length - 24; + if (excess <= 0) return Number(value); + const kept = magnitude >> BigInt(excess); + const rest = magnitude - (kept << BigInt(excess)); + const half = 1n << BigInt(excess - 1); + const rounded = Number(rest > half || (rest === half && (kept & 1n) === 1n) ? kept + 1n : kept) * 2 ** excess; + return value < 0n ? -rounded : rounded; +} + function floatCodec(typeName: string, wide: boolean): Codec { const what = wide ? "a float64" : "a float32"; return { @@ -98,7 +111,8 @@ function floatCodec(typeName: string, wide: boolean): Codec { }, decode(raw) { if (raw instanceof WireFloat) return wide ? raw.value : Math.fround(raw.value); - if (typeof raw === "number" || typeof raw === "bigint") return wide ? Number(raw) : Math.fround(Number(raw)); + if (typeof raw === "number") return wide ? raw : Math.fround(raw); + if (typeof raw === "bigint") return wide ? Number(raw) : bigintToFloat32(raw); return refuse(what, kindOf(raw)); }, }; @@ -170,21 +184,25 @@ type SlotSpec = type SlotsSpec = Readonly }[keyof TValue & string]>>; +const INT32_MIN = -(2 ** 31); const INT32_MAX = 2 ** 31 - 1; -// The declared keys, ascending. An arm key is an Int32 in the reference, so a union refuses a wider key at definition. -function slotKeys(spec: Readonly>, owner: string, max = Number.MAX_SAFE_INTEGER): number[] { - // Object.keys lists integer keys in ascending order, so the declared keys come back sorted. - return Object.keys(spec).map((text) => { - const key = Number(text); - if (!Number.isInteger(key) || key < 0 || String(key) !== text) { - throw new Error(`${owner} declares the key "${text}", which is not a non-negative integer.`); - } - if (key > max) { - throw new Error(`${owner} declares the key ${key}, which is outside Int32.`); - } - return key; - }); +// The declared keys, ascending. The reference's keys are Int32: a union arm may be any (a negative one too), a slot is +// non-negative, and a key outside the range is refused at definition rather than allocated for. +function slotKeys(spec: Readonly>, owner: string, low: number): number[] { + return Object.keys(spec) + .map((text) => { + if (!/^-?(0|[1-9][0-9]*)$/u.test(text) || (low >= 0 && text.startsWith("-"))) { + throw new Error(`${owner} declares the key "${text}", which is not ${low < 0 ? "an" : "a non-negative"} integer.`); + } + const key = BigInt(text); + if (key < BigInt(low) || key > BigInt(INT32_MAX)) { + throw new Error(`${owner} declares the key ${text}, which is outside Int32.`); + } + return Number(key); + }) + // Object.keys lists negative keys after the others, in the order they were written: sort them numerically. + .sort((left, right) => left - right); } function pascalCase(name: string): string { @@ -199,7 +217,7 @@ function pascalCase(name: string): string { // are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that // declares a default also takes the default (cultcache-py's rule). export function slots(spec: SlotsSpec, typeName?: string): SlotsCodec { - const keys = slotKeys(spec, "A slot record"); + const keys = slotKeys(spec, "A slot record", 0); if (keys.length === 0) { throw new Error("A slot record declares no slots."); } @@ -271,7 +289,7 @@ export function union( arms: Readonly>>, typeName?: string, ): Codec { - const keys = slotKeys(arms, `Union ${name}`, INT32_MAX); + const keys = slotKeys(arms, `Union ${name}`, INT32_MIN); if (keys.length === 0) { throw new Error(`Union ${name} declares no arms.`); } @@ -314,7 +332,7 @@ export function union( if (typeof key !== "number" && typeof key !== "bigint") { throw new Error(`Union ${name}: the arm key is not an integer (declared keys: ${declared}).`); } - if (key < -(2 ** 31) || key > INT32_MAX) { + if (key < INT32_MIN || key > INT32_MAX) { throw new Error(`Union ${name}: the arm key is outside Int32 (declared keys: ${declared}).`); } const arm = byKey.get(Number(key)); diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 78e9c7c0..75a94f58 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,7 @@ export * as codecs from "./codec"; export type { Codec, SlotsCodec } from "./codec"; export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { defineDocumentRegistry, defineDocumentType } from "./document"; +export { defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; export type { AnyCultCacheDocumentDefinition, diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index cb76ed03..2b4d4dfc 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -12,7 +12,6 @@ import { codecs } from "../src"; import type { Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; -import { referenceReads, referenceWrites } from "./reference-numeric-table"; import { defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; @@ -378,7 +377,7 @@ test("slot names become catalog member names in Pascal case", () => { test("definitions are refused at definition time", () => { assert.throws(() => union<{ kind: "A" }>("U", { 0: ["A", slots<{ kind: string }>({ 0: ["kind", string] })] }), /Union U arm "A" declares a slot named kind/u); assert.throws(() => union<{ kind: "A" | "B" }>("U", { 0: ["A", unit], 1: ["A" as "B", unit] }), /declares the arm "A" twice/u); - assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not a non-negative integer\.$/u); + assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not an integer\.$/u); assert.throws(() => union<{ kind: "A" }>("U", { "01": ["A", unit] } as never), /key "01"/u); assert.throws(() => union<{ kind: "A" }>("U", { "1.5": ["A", unit] } as never), /key "1\.5"/u); assert.throws(() => union<{ kind: "A" }>("U", {} as never), /^Error: Union U declares no arms\.$/u); @@ -532,39 +531,55 @@ test("every leaf and wrapper carries the reference's type name, and a catalog is assert.equal(given.members!.length, 1); }); -// referenceReads and referenceWrites are the C# reference's own answers (see the table's header): a numeric leaf reads -// and refuses exactly what the reference does, and writes exactly the bytes the reference reads back unchanged. +// tests/vectors/numeric-leaves is written by the C# reference (NumericLeafVectorTests): what it reads and refuses for each +// numeric leaf, and the bytes it writes. +const numericVectors = join(vectors, "..", "numeric-leaves"); const numericLeaves: Record> = { int32, int64, float32, float64 }; -const bitsOf = (value: number) => (Number.isNaN(value) ? "NaN" : Buffer.from(new Float64Array([value]).buffer).reverse().toString("hex")); +const doubleBitsHex = (value: number) => { + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value); + return Buffer.from(view.buffer).toString("hex"); +}; +const int64BitsHex = (value: number | bigint) => BigInt.asUintN(64, BigInt(value)).toString(16).padStart(16, "0"); +const isNaNBits = (bitsHex: string) => { + const bits = BigInt(`0x${bitsHex}`); + return ((bits >> 52n) & 0x7ffn) === 0x7ffn && (bits & ((1n << 52n) - 1n)) !== 0n; +}; -test("numeric leaves read and refuse exactly what the C# reference does", () => { - assert.equal(referenceReads.length, 120); +test("numeric leaves read and refuse exactly what the C# reference does", async () => { + const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "reads.msgpack")))) as [string, Uint8Array, boolean, Uint8Array][]; + assert.equal(rows.length, 148); let accepted = 0; let refused = 0; - for (const [leaf, value, outcome] of referenceReads) { + for (const [leaf, payload, wasAccepted, bits] of rows) { + const label = `${leaf} ${hex(payload)}`; const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); - let got: string; - try { - const read = record.decode(readWire(fromHex(`91${value}`))).v; - got = `ACCEPT ${leaf.startsWith("float") ? bitsOf(read as number) : String(read)}`; + if (wasAccepted) { + const read = record.decode(readWire(payload)).v as number | bigint; + const wanted = hex(bits); + if (leaf.startsWith("float")) { + if (isNaNBits(wanted)) assert.ok(Number.isNaN(read), label); + else assert.equal(doubleBitsHex(read as number), wanted, label); + } else { + assert.equal(int64BitsHex(read), wanted, label); + } accepted += 1; - } catch (error) { - assert.ok(!(error as Error).message.includes(value), "a refusal never echoes the value"); - got = "REFUSE"; + } else { + assert.throws(() => record.decode(readWire(payload)), (error: Error) => !error.message.includes(hex(payload).slice(2)), label); refused += 1; } - assert.equal(got, outcome, `${leaf} ${value}`); } - assert.deepEqual([accepted, refused], [referenceReads.filter((row) => row[2].startsWith("ACCEPT")).length, referenceReads.filter((row) => row[2] === "REFUSE").length]); - assert.ok(accepted > 40 && refused > 40); + assert.ok(accepted > 40 && refused > 40, `accepted ${accepted}, refused ${refused}`); + assert.equal(accepted + refused, rows.length); }); -test("numeric leaves write the bytes the C# reference reads back unchanged, and 64-bit values lose nothing", () => { - assert.equal(referenceWrites.length, 46); - for (const [leaf, text, expected] of referenceWrites) { - const value = leaf.startsWith("float") ? (text === "-0" ? -0 : Number(text)) : text.endsWith("n") || Math.abs(Number(text)) > Number.MAX_SAFE_INTEGER ? BigInt(text.replace("n", "")) : Number(text); +test("numeric leaves write the bytes the C# reference writes, and 64-bit values lose nothing", async () => { + const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "writes.msgpack")))) as [string, string, Uint8Array][]; + assert.equal(rows.length, 46); + for (const [leaf, text, expected] of rows) { + const value = leaf.startsWith("float") ? new DataView(Uint8Array.from(Buffer.from(text, "hex")).buffer).getFloat64(0) : leaf === "int32" ? Number(text) : BigInt(text); const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); - assert.equal(hex(writeWire(record.encode({ v: value }))), expected, `${leaf} ${text}`); + assert.equal(hex(writeWire(record.encode({ v: value }))), hex(expected), `${leaf} ${text}`); } const wide = slots<{ v: number | bigint }>({ 0: ["v", int64] }); for (const value of [2n ** 53n + 1n, 2n ** 63n - 1n, -(2n ** 63n), -(2n ** 53n) - 1n]) { @@ -642,3 +657,32 @@ function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entrie async delete() {}, }; } + +test("arm keys span Int32 and are listed ascending; slot keys are non-negative Int32", () => { + const keys = union<{ kind: "High" | "Zero" | "Neg" | "Low" }>("U", { + 2147483647: ["High", unit], + 0: ["Zero", unit], + [-1]: ["Neg", unit], + [-2147483648]: ["Low", unit], + }); + for (const [kind, key] of [["High", 2147483647], ["Zero", 0], ["Neg", -1], ["Low", -2147483648]] as const) { + const written = keys.encode({ kind }); + assert.deepEqual(written, [key, []]); + assert.deepEqual(keys.decode(readWire(writeWire(written))), { kind }); + } + assert.throws(() => keys.decode(readWire(fromHex("92059090"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." }); + assert.throws(() => union<{ kind: "A" }>("U", { [-2147483649]: ["A", unit] }), { message: "Union U declares the key -2147483649, which is outside Int32." }); + assert.throws(() => union<{ kind: "A" }>("U", { 99999999999999999999: ["A", unit] } as never), { message: "Union U declares the key 100000000000000000000, which is outside Int32." }); + assert.doesNotThrow(() => slots<{ a: string }>({ 2147483647: ["a", string] })); + assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is outside Int32." }); + assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), { message: 'A slot record declares the key "-1", which is not a non-negative integer.' }); +}); + +test("the mixed-record vector is the bytes the reference writes for a record of every leaf kind", async () => { + const mixed = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({ + 0: ["id", string], 1: ["i", int32], 2: ["l", int64], 3: ["f", float32], 4: ["d", float64], + }); + const committed = new Uint8Array(await readFile(join(vectors, "..", "numeric-leaves", "mixed-record.msgpack"))); + assert.equal(hex(writeWire(mixed.encode({ id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }))), hex(committed)); + assert.deepEqual(mixed.decode(readWire(committed)), { id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }); +}); diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index a2eda250..fb970906 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -1,12 +1,11 @@ -import { decode, encode } from "@msgpack/msgpack"; import type { AnyCultCacheDocumentDefinition, CultCache, CultCacheDocumentDefinition, - CultCacheDocumentFormatter, CultCacheDocumentValue, CultCacheEnvelope, } from "@gamecult/cultcache-ts"; +import { resolveDocumentFormatter } from "@gamecult/cultcache-ts"; import { type CultNetDocumentDeleteMessage, @@ -136,7 +135,7 @@ export class CultNetDocumentRegistry { } = {}, ): CultNetDocumentPutRawMessage { const parsed = binding.definition.schema.parse(value); - const formatter = formatterFor(binding.definition); + const formatter = resolveDocumentFormatter(binding.definition); return { schemaVersion: "cultnet.document_put_raw.v0", messageId, @@ -435,18 +434,7 @@ function decodeDocumentValue( definition: TDefinition, payload: Uint8Array, ): CultCacheDocumentValue { - const formatter = formatterFor(definition); + const formatter = resolveDocumentFormatter(definition); const decoded = formatter.decode(payload); return definition.schema.parse(decoded) as CultCacheDocumentValue; } - -function formatterFor( - definition: TDefinition, -): CultCacheDocumentFormatter> { - return (definition.formatter as CultCacheDocumentFormatter< - CultCacheDocumentValue - > | undefined) ?? { - encode: (value: CultCacheDocumentValue) => encode(value), - decode: (bytes: Uint8Array) => decode(bytes) as CultCacheDocumentValue, - }; -} diff --git a/packages/cultnet-ts/test/replication-codec.test.ts b/packages/cultnet-ts/test/replication-codec.test.ts new file mode 100644 index 00000000..0c6cd632 --- /dev/null +++ b/packages/cultnet-ts/test/replication-codec.test.ts @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +import { z } from "zod"; +import { CultCache, codecs, defineDocumentType, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; + +import { CultNetDocumentRegistry, defineCultNetDocumentBinding } from "../src"; + +const { float32, float64, int32, int64, slots, string } = codecs; + +// tests/vectors/numeric-leaves/mixed-record.msgpack is the bytes the C# reference writes for Mixed { "n1", 7, 8, 1.5f, 2.5 } +// (NumericLeafVectorTests). A payload equal to it is a payload the reference reads. +function referenceMixedRecord(): Uint8Array { + for (let dir = __dirname; ; dir = resolve(dir, "..")) { + const file = join(dir, "tests", "vectors", "numeric-leaves", "mixed-record.msgpack"); + if (existsSync(file)) return new Uint8Array(readFileSync(file)); + if (resolve(dir, "..") === dir) throw new Error("tests/vectors/numeric-leaves not found above the test."); + } +} + +const mixedCodec = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({ + 0: ["id", string], + 1: ["i", int32], + 2: ["l", int64], + 3: ["f", float32], + 4: ["d", float64], +}); + +const mixedDefinition = defineDocumentType({ + type: "tests.mixed_record", + schemaId: "tests.mixed_record.v1", + schemaName: "tests.mixed_record", + schemaVersion: "tests.mixed_record.v1", + schema: z.object({ id: z.string(), i: z.number(), l: z.union([z.number(), z.bigint()]), f: z.number(), d: z.number() }), + codec: mixedCodec, + name: "id", +}); + +const value = { id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }; +const hex = (data: Uint8Array) => Buffer.from(data).toString("hex"); + +function registry() { + return new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: mixedDefinition })]); +} + +test("a codec document replicates as the bytes the C# reference writes, and a snapshot of it applies on another cache", async () => { + const dir = mkdtempSync(join(tmpdir(), "cultnet-codec-")); + try { + const reference = referenceMixedRecord(); + const documents = registry(); + const origin = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "origin.cc"))).build(); + const target = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "target.cc"))).build(); + + const put = documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-1", "n1", value); + assert.equal(hex(put.document.payload), hex(reference), "a raw put carries the reference's record, not a map"); + + await documents.applyRawDocumentPutMessage(origin, put); + assert.equal(hex(origin.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + + const raw = documents.createRawSnapshotResponse(origin, "snapshot-raw"); + assert.equal(hex(raw.documents[0].payload), hex(reference)); + await documents.applyRawSnapshotResponse(target, raw); + assert.deepEqual(target.getRequired(mixedDefinition, "n1"), value); + assert.equal(hex(target.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + + // The value-level snapshot decodes the stored record through the codec, so a peer applying it parses a record. + const values = documents.createSnapshotResponse(origin, "snapshot-values"); + assert.deepEqual(values.documents[0].payload, value); + const other = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "other.cc"))).build(); + await documents.applySnapshotResponse(other, values); + assert.equal(hex(other.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a definition that carries a codec and a formatter is refused where replication reads it", () => { + const both = { + ...mixedDefinition, + formatter: { encode: () => new Uint8Array(), decode: () => value }, + } as unknown as typeof mixedDefinition; + const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: both })]); + assert.throws( + () => documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-2", "n1", value), + { message: 'CultCache document type "tests.mixed_record" declares both a codec and a formatter.' }, + ); +}); + +test("a document with neither a codec nor a formatter replicates as a map, as before", () => { + const plain = defineDocumentType({ + type: "tests.plain_record", + schemaId: "tests.plain_record.v1", + schemaName: "tests.plain_record", + schemaVersion: "tests.plain_record.v1", + schema: z.object({ id: z.string() }), + name: "id", + }); + const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: plain })]); + const put = documents.createRawDocumentPutMessage(documents.get("tests.plain_record")!, "put-3", "n1", { id: "n1" }); + assert.equal(hex(put.document.payload), "81a26964a26e31"); +});