cultcache-ts and cultnet-ts: replication writes a codec document through the codec; numeric oracle read from the C# vectors; float32 rounds once; arm and slot keys span the reference's Int32

- cultnet-ts replication calls cultcache-ts's resolveDocumentFormatter (now exported) and loses its private
  @msgpack/msgpack default, so a raw put and a snapshot of a codec document carry the codec's bytes.
- the hand-written numeric table is deleted; the tests read tests/vectors/numeric-leaves.
- an integer past 2^53 in a float32 slot rounds once (ties to even), not through a double.
- a union arm key is any Int32 (negative too, sorted numerically); a slot key is 0 to Int32 max.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-11 08:34:31 +02:00
commit fa1c006bee
6 changed files with 213 additions and 59 deletions

View file

@ -272,13 +272,13 @@ const sceneDocument = defineDocumentType({
- The leaves are `string`, `boolean`, `bytes`, `int32`, `int64`, `float32` and `float64`, each writing the MessagePack the
C# type writes and refusing what C# refuses (a float in an int slot, an `int` outside Int32). `int64` reads a number
when the value is a safe integer and a bigint beyond 2^53, and refuses a number beyond 2^53 on write. `list`, `array`
and `nullable` wrap a leaf, a union or a record.
and `nullable` wrap a leaf, a union or a record. A `float32` slot rounds an integer once, as the reference's cast does.
- Every codec carries the reference's type name, and a store's schema catalog is derived from those names, because the
reference refuses a store whose slot names another type. The leaves know theirs; give a union or a record its C# name
as the last argument of `union(name, arms, typeName)` or `slots(spec, typeName)`. A codec with a slot it cannot name
refuses to derive members: pass `members` instead.
- An arm key is an integer within Int32, in any MessagePack integer encoding. A float key (even `1.0`) is refused on
read, and a union refuses a key outside Int32 when it is defined.
- An arm key is an integer within Int32 (negative too), in any MessagePack integer encoding. A float key (even `1.0`) is
refused on read, and a union refuses a key outside Int32 when it is defined. A slot key is within 0 to Int32 max.
- `codec` supplies the document's formatter and, unless `members` is given, its catalog members. A definition may not
carry both `codec` and `formatter`. `schema.parse` still runs on every decoded value.

View file

@ -87,6 +87,19 @@ export const int32 = integerCodec("System.Int32", "an int32", -(2n ** 31n), 2n *
// C# long. A number when the value is a safe integer, a bigint beyond 2^53 (so 64-bit values never lose precision).
export const int64 = integerCodec("System.Int64", "an int64", -(2n ** 63n), 2n ** 63n - 1n);
// A 64-bit integer to the nearest float32 in one rounding, ties to even, as the reference's cast does. Number(value)
// first would round to a double and then again, and the two roundings can disagree.
function bigintToFloat32(value: bigint): number {
const magnitude = value < 0n ? -value : value;
const excess = magnitude.toString(2).length - 24;
if (excess <= 0) return Number(value);
const kept = magnitude >> BigInt(excess);
const rest = magnitude - (kept << BigInt(excess));
const half = 1n << BigInt(excess - 1);
const rounded = Number(rest > half || (rest === half && (kept & 1n) === 1n) ? kept + 1n : kept) * 2 ** excess;
return value < 0n ? -rounded : rounded;
}
function floatCodec(typeName: string, wide: boolean): Codec<number> {
const what = wide ? "a float64" : "a float32";
return {
@ -98,7 +111,8 @@ function floatCodec(typeName: string, wide: boolean): Codec<number> {
},
decode(raw) {
if (raw instanceof WireFloat) return wide ? raw.value : Math.fround(raw.value);
if (typeof raw === "number" || typeof raw === "bigint") return wide ? Number(raw) : Math.fround(Number(raw));
if (typeof raw === "number") return wide ? raw : Math.fround(raw);
if (typeof raw === "bigint") return wide ? Number(raw) : bigintToFloat32(raw);
return refuse(what, kindOf(raw));
},
};
@ -170,21 +184,25 @@ type SlotSpec<TValue, TName extends string = string> =
type SlotsSpec<TValue> = Readonly<Record<number, { [K in keyof TValue & string]: SlotSpec<TValue[K], K> }[keyof TValue & string]>>;
const INT32_MIN = -(2 ** 31);
const INT32_MAX = 2 ** 31 - 1;
// The declared keys, ascending. An arm key is an Int32 in the reference, so a union refuses a wider key at definition.
function slotKeys(spec: Readonly<Record<string, unknown>>, owner: string, max = Number.MAX_SAFE_INTEGER): number[] {
// Object.keys lists integer keys in ascending order, so the declared keys come back sorted.
return Object.keys(spec).map((text) => {
const key = Number(text);
if (!Number.isInteger(key) || key < 0 || String(key) !== text) {
throw new Error(`${owner} declares the key "${text}", which is not a non-negative integer.`);
}
if (key > max) {
throw new Error(`${owner} declares the key ${key}, which is outside Int32.`);
}
return key;
});
// The declared keys, ascending. The reference's keys are Int32: a union arm may be any (a negative one too), a slot is
// non-negative, and a key outside the range is refused at definition rather than allocated for.
function slotKeys(spec: Readonly<Record<string, unknown>>, owner: string, low: number): number[] {
return Object.keys(spec)
.map((text) => {
if (!/^-?(0|[1-9][0-9]*)$/u.test(text) || (low >= 0 && text.startsWith("-"))) {
throw new Error(`${owner} declares the key "${text}", which is not ${low < 0 ? "an" : "a non-negative"} integer.`);
}
const key = BigInt(text);
if (key < BigInt(low) || key > BigInt(INT32_MAX)) {
throw new Error(`${owner} declares the key ${text}, which is outside Int32.`);
}
return Number(key);
})
// Object.keys lists negative keys after the others, in the order they were written: sort them numerically.
.sort((left, right) => left - right);
}
function pascalCase(name: string): string {
@ -199,7 +217,7 @@ function pascalCase(name: string): string {
// are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that
// declares a default also takes the default (cultcache-py's rule).
export function slots<TValue extends object>(spec: SlotsSpec<TValue>, typeName?: string): SlotsCodec<TValue> {
const keys = slotKeys(spec, "A slot record");
const keys = slotKeys(spec, "A slot record", 0);
if (keys.length === 0) {
throw new Error("A slot record declares no slots.");
}
@ -271,7 +289,7 @@ export function union<TValue extends { kind: string }>(
arms: Readonly<Record<number, ArmSpec<TValue["kind"]>>>,
typeName?: string,
): Codec<TValue | null> {
const keys = slotKeys(arms, `Union ${name}`, INT32_MAX);
const keys = slotKeys(arms, `Union ${name}`, INT32_MIN);
if (keys.length === 0) {
throw new Error(`Union ${name} declares no arms.`);
}
@ -314,7 +332,7 @@ export function union<TValue extends { kind: string }>(
if (typeof key !== "number" && typeof key !== "bigint") {
throw new Error(`Union ${name}: the arm key is not an integer (declared keys: ${declared}).`);
}
if (key < -(2 ** 31) || key > INT32_MAX) {
if (key < INT32_MIN || key > INT32_MAX) {
throw new Error(`Union ${name}: the arm key is outside Int32 (declared keys: ${declared}).`);
}
const arm = byKey.get(Number(key));

View file

@ -2,7 +2,7 @@ export * as codecs from "./codec";
export type { Codec, SlotsCodec } from "./codec";
export { CultCache, CultCacheBuilder } from "./cult-cache";
export { inspectCultCacheBytes } from "./cult-cache-inspector";
export { defineDocumentRegistry, defineDocumentType } from "./document";
export { defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document";
export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store";
export type {
AnyCultCacheDocumentDefinition,

View file

@ -12,7 +12,6 @@ import { codecs } from "../src";
import type { Codec } from "../src/codec";
import { CultCache } from "../src/cult-cache";
import { readWire, WireFloat, writeWire } from "../src/wire";
import { referenceReads, referenceWrites } from "./reference-numeric-table";
import { defineDocumentType } from "../src/document";
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types";
@ -378,7 +377,7 @@ test("slot names become catalog member names in Pascal case", () => {
test("definitions are refused at definition time", () => {
assert.throws(() => union<{ kind: "A" }>("U", { 0: ["A", slots<{ kind: string }>({ 0: ["kind", string] })] }), /Union U arm "A" declares a slot named kind/u);
assert.throws(() => union<{ kind: "A" | "B" }>("U", { 0: ["A", unit], 1: ["A" as "B", unit] }), /declares the arm "A" twice/u);
assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not a non-negative integer\.$/u);
assert.throws(() => union<{ kind: "A" }>("U", { x: ["A", unit] } as never), /^Error: Union U declares the key "x", which is not an integer\.$/u);
assert.throws(() => union<{ kind: "A" }>("U", { "01": ["A", unit] } as never), /key "01"/u);
assert.throws(() => union<{ kind: "A" }>("U", { "1.5": ["A", unit] } as never), /key "1\.5"/u);
assert.throws(() => union<{ kind: "A" }>("U", {} as never), /^Error: Union U declares no arms\.$/u);
@ -532,39 +531,55 @@ test("every leaf and wrapper carries the reference's type name, and a catalog is
assert.equal(given.members!.length, 1);
});
// referenceReads and referenceWrites are the C# reference's own answers (see the table's header): a numeric leaf reads
// and refuses exactly what the reference does, and writes exactly the bytes the reference reads back unchanged.
// tests/vectors/numeric-leaves is written by the C# reference (NumericLeafVectorTests): what it reads and refuses for each
// numeric leaf, and the bytes it writes.
const numericVectors = join(vectors, "..", "numeric-leaves");
const numericLeaves: Record<string, Codec<unknown>> = { int32, int64, float32, float64 };
const bitsOf = (value: number) => (Number.isNaN(value) ? "NaN" : Buffer.from(new Float64Array([value]).buffer).reverse().toString("hex"));
const doubleBitsHex = (value: number) => {
const view = new DataView(new ArrayBuffer(8));
view.setFloat64(0, value);
return Buffer.from(view.buffer).toString("hex");
};
const int64BitsHex = (value: number | bigint) => BigInt.asUintN(64, BigInt(value)).toString(16).padStart(16, "0");
const isNaNBits = (bitsHex: string) => {
const bits = BigInt(`0x${bitsHex}`);
return ((bits >> 52n) & 0x7ffn) === 0x7ffn && (bits & ((1n << 52n) - 1n)) !== 0n;
};
test("numeric leaves read and refuse exactly what the C# reference does", () => {
assert.equal(referenceReads.length, 120);
test("numeric leaves read and refuse exactly what the C# reference does", async () => {
const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "reads.msgpack")))) as [string, Uint8Array, boolean, Uint8Array][];
assert.equal(rows.length, 148);
let accepted = 0;
let refused = 0;
for (const [leaf, value, outcome] of referenceReads) {
for (const [leaf, payload, wasAccepted, bits] of rows) {
const label = `${leaf} ${hex(payload)}`;
const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] });
let got: string;
try {
const read = record.decode(readWire(fromHex(`91${value}`))).v;
got = `ACCEPT ${leaf.startsWith("float") ? bitsOf(read as number) : String(read)}`;
if (wasAccepted) {
const read = record.decode(readWire(payload)).v as number | bigint;
const wanted = hex(bits);
if (leaf.startsWith("float")) {
if (isNaNBits(wanted)) assert.ok(Number.isNaN(read), label);
else assert.equal(doubleBitsHex(read as number), wanted, label);
} else {
assert.equal(int64BitsHex(read), wanted, label);
}
accepted += 1;
} catch (error) {
assert.ok(!(error as Error).message.includes(value), "a refusal never echoes the value");
got = "REFUSE";
} else {
assert.throws(() => record.decode(readWire(payload)), (error: Error) => !error.message.includes(hex(payload).slice(2)), label);
refused += 1;
}
assert.equal(got, outcome, `${leaf} ${value}`);
}
assert.deepEqual([accepted, refused], [referenceReads.filter((row) => row[2].startsWith("ACCEPT")).length, referenceReads.filter((row) => row[2] === "REFUSE").length]);
assert.ok(accepted > 40 && refused > 40);
assert.ok(accepted > 40 && refused > 40, `accepted ${accepted}, refused ${refused}`);
assert.equal(accepted + refused, rows.length);
});
test("numeric leaves write the bytes the C# reference reads back unchanged, and 64-bit values lose nothing", () => {
assert.equal(referenceWrites.length, 46);
for (const [leaf, text, expected] of referenceWrites) {
const value = leaf.startsWith("float") ? (text === "-0" ? -0 : Number(text)) : text.endsWith("n") || Math.abs(Number(text)) > Number.MAX_SAFE_INTEGER ? BigInt(text.replace("n", "")) : Number(text);
test("numeric leaves write the bytes the C# reference writes, and 64-bit values lose nothing", async () => {
const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "writes.msgpack")))) as [string, string, Uint8Array][];
assert.equal(rows.length, 46);
for (const [leaf, text, expected] of rows) {
const value = leaf.startsWith("float") ? new DataView(Uint8Array.from(Buffer.from(text, "hex")).buffer).getFloat64(0) : leaf === "int32" ? Number(text) : BigInt(text);
const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] });
assert.equal(hex(writeWire(record.encode({ v: value }))), expected, `${leaf} ${text}`);
assert.equal(hex(writeWire(record.encode({ v: value }))), hex(expected), `${leaf} ${text}`);
}
const wide = slots<{ v: number | bigint }>({ 0: ["v", int64] });
for (const value of [2n ** 53n + 1n, 2n ** 63n - 1n, -(2n ** 63n), -(2n ** 53n) - 1n]) {
@ -642,3 +657,32 @@ function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entrie
async delete() {},
};
}
test("arm keys span Int32 and are listed ascending; slot keys are non-negative Int32", () => {
const keys = union<{ kind: "High" | "Zero" | "Neg" | "Low" }>("U", {
2147483647: ["High", unit],
0: ["Zero", unit],
[-1]: ["Neg", unit],
[-2147483648]: ["Low", unit],
});
for (const [kind, key] of [["High", 2147483647], ["Zero", 0], ["Neg", -1], ["Low", -2147483648]] as const) {
const written = keys.encode({ kind });
assert.deepEqual(written, [key, []]);
assert.deepEqual(keys.decode(readWire(writeWire(written))), { kind });
}
assert.throws(() => keys.decode(readWire(fromHex("92059090"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." });
assert.throws(() => union<{ kind: "A" }>("U", { [-2147483649]: ["A", unit] }), { message: "Union U declares the key -2147483649, which is outside Int32." });
assert.throws(() => union<{ kind: "A" }>("U", { 99999999999999999999: ["A", unit] } as never), { message: "Union U declares the key 100000000000000000000, which is outside Int32." });
assert.doesNotThrow(() => slots<{ a: string }>({ 2147483647: ["a", string] }));
assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is outside Int32." });
assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), { message: 'A slot record declares the key "-1", which is not a non-negative integer.' });
});
test("the mixed-record vector is the bytes the reference writes for a record of every leaf kind", async () => {
const mixed = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({
0: ["id", string], 1: ["i", int32], 2: ["l", int64], 3: ["f", float32], 4: ["d", float64],
});
const committed = new Uint8Array(await readFile(join(vectors, "..", "numeric-leaves", "mixed-record.msgpack")));
assert.equal(hex(writeWire(mixed.encode({ id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 }))), hex(committed));
assert.deepEqual(mixed.decode(readWire(committed)), { id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 });
});

View file

@ -1,12 +1,11 @@
import { decode, encode } from "@msgpack/msgpack";
import type {
AnyCultCacheDocumentDefinition,
CultCache,
CultCacheDocumentDefinition,
CultCacheDocumentFormatter,
CultCacheDocumentValue,
CultCacheEnvelope,
} from "@gamecult/cultcache-ts";
import { resolveDocumentFormatter } from "@gamecult/cultcache-ts";
import {
type CultNetDocumentDeleteMessage,
@ -136,7 +135,7 @@ export class CultNetDocumentRegistry {
} = {},
): CultNetDocumentPutRawMessage {
const parsed = binding.definition.schema.parse(value);
const formatter = formatterFor(binding.definition);
const formatter = resolveDocumentFormatter(binding.definition);
return {
schemaVersion: "cultnet.document_put_raw.v0",
messageId,
@ -435,18 +434,7 @@ function decodeDocumentValue<TDefinition extends CultCacheDocumentDefinition>(
definition: TDefinition,
payload: Uint8Array,
): CultCacheDocumentValue<TDefinition> {
const formatter = formatterFor(definition);
const formatter = resolveDocumentFormatter(definition);
const decoded = formatter.decode(payload);
return definition.schema.parse(decoded) as CultCacheDocumentValue<TDefinition>;
}
function formatterFor<TDefinition extends CultCacheDocumentDefinition>(
definition: TDefinition,
): CultCacheDocumentFormatter<CultCacheDocumentValue<TDefinition>> {
return (definition.formatter as CultCacheDocumentFormatter<
CultCacheDocumentValue<TDefinition>
> | undefined) ?? {
encode: (value: CultCacheDocumentValue<TDefinition>) => encode(value),
decode: (bytes: Uint8Array) => decode(bytes) as CultCacheDocumentValue<TDefinition>,
};
}

View file

@ -0,0 +1,104 @@
import assert from "node:assert/strict";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import test from "node:test";
import { z } from "zod";
import { CultCache, codecs, defineDocumentType, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts";
import { CultNetDocumentRegistry, defineCultNetDocumentBinding } from "../src";
const { float32, float64, int32, int64, slots, string } = codecs;
// tests/vectors/numeric-leaves/mixed-record.msgpack is the bytes the C# reference writes for Mixed { "n1", 7, 8, 1.5f, 2.5 }
// (NumericLeafVectorTests). A payload equal to it is a payload the reference reads.
function referenceMixedRecord(): Uint8Array {
for (let dir = __dirname; ; dir = resolve(dir, "..")) {
const file = join(dir, "tests", "vectors", "numeric-leaves", "mixed-record.msgpack");
if (existsSync(file)) return new Uint8Array(readFileSync(file));
if (resolve(dir, "..") === dir) throw new Error("tests/vectors/numeric-leaves not found above the test.");
}
}
const mixedCodec = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({
0: ["id", string],
1: ["i", int32],
2: ["l", int64],
3: ["f", float32],
4: ["d", float64],
});
const mixedDefinition = defineDocumentType({
type: "tests.mixed_record",
schemaId: "tests.mixed_record.v1",
schemaName: "tests.mixed_record",
schemaVersion: "tests.mixed_record.v1",
schema: z.object({ id: z.string(), i: z.number(), l: z.union([z.number(), z.bigint()]), f: z.number(), d: z.number() }),
codec: mixedCodec,
name: "id",
});
const value = { id: "n1", i: 7, l: 8, f: 1.5, d: 2.5 };
const hex = (data: Uint8Array) => Buffer.from(data).toString("hex");
function registry() {
return new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: mixedDefinition })]);
}
test("a codec document replicates as the bytes the C# reference writes, and a snapshot of it applies on another cache", async () => {
const dir = mkdtempSync(join(tmpdir(), "cultnet-codec-"));
try {
const reference = referenceMixedRecord();
const documents = registry();
const origin = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "origin.cc"))).build();
const target = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "target.cc"))).build();
const put = documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-1", "n1", value);
assert.equal(hex(put.document.payload), hex(reference), "a raw put carries the reference's record, not a map");
await documents.applyRawDocumentPutMessage(origin, put);
assert.equal(hex(origin.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference));
const raw = documents.createRawSnapshotResponse(origin, "snapshot-raw");
assert.equal(hex(raw.documents[0].payload), hex(reference));
await documents.applyRawSnapshotResponse(target, raw);
assert.deepEqual(target.getRequired(mixedDefinition, "n1"), value);
assert.equal(hex(target.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference));
// The value-level snapshot decodes the stored record through the codec, so a peer applying it parses a record.
const values = documents.createSnapshotResponse(origin, "snapshot-values");
assert.deepEqual(values.documents[0].payload, value);
const other = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "other.cc"))).build();
await documents.applySnapshotResponse(other, values);
assert.equal(hex(other.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference));
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("a definition that carries a codec and a formatter is refused where replication reads it", () => {
const both = {
...mixedDefinition,
formatter: { encode: () => new Uint8Array(), decode: () => value },
} as unknown as typeof mixedDefinition;
const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: both })]);
assert.throws(
() => documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-2", "n1", value),
{ message: 'CultCache document type "tests.mixed_record" declares both a codec and a formatter.' },
);
});
test("a document with neither a codec nor a formatter replicates as a map, as before", () => {
const plain = defineDocumentType({
type: "tests.plain_record",
schemaId: "tests.plain_record.v1",
schemaName: "tests.plain_record",
schemaVersion: "tests.plain_record.v1",
schema: z.object({ id: z.string() }),
name: "id",
});
const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: plain })]);
const put = documents.createRawDocumentPutMessage(documents.get("tests.plain_record")!, "put-3", "n1", { id: "n1" });
assert.equal(hex(put.document.payload), "81a26964a26e31");
});