From 1e0c5a920977a0bdd6b4fc27facb757eae7cd022 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 11 Oct 2026 09:18:18 +0200 Subject: [PATCH] batch 4 code: mesh readers decode through the definition's formatter, float32 sNaN refused, slot keys capped at 65535 cultmesh-ts reads document payloads through resolveDocumentFormatter at its five plain-msgpack sites; WireFloat carries a signalling flag and the float32 leaf refuses it; slots() refuses a key above 65535 at definition. The C# vector writer gains the 2^24+1 reads row and float32-nan.msgpack, whose committed files land in the next commit: this commit does not build its tests (union-parity numeric vectors, row count 160 and float32-nan.msgpack are absent). Co-Authored-By: Claude Opus 5.5 --- packages/cultcache-ts/src/codec.ts | 22 +++-- packages/cultcache-ts/src/wire.ts | 9 +- .../cultcache-ts/test/union-parity.test.ts | 54 +++++++++- packages/cultmesh-ts/src/index.ts | 27 +++-- .../cultmesh-ts/test/codec-documents.test.ts | 98 +++++++++++++++++++ .../NumericLeafVectorTests.cs | 37 +++++++ 6 files changed, 226 insertions(+), 21 deletions(-) create mode 100644 packages/cultmesh-ts/test/codec-documents.test.ts diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 1ad50f8e..7dba8edc 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -110,7 +110,11 @@ function floatCodec(typeName: string, wide: boolean): Codec { return new WireFloat(value, wide); }, decode(raw) { - if (raw instanceof WireFloat) return wide ? raw.value : Math.fround(raw.value); + if (raw instanceof WireFloat) { + // A JS number has no NaN payload, so a signalling float32 NaN cannot be kept; refusing beats changing its bytes. + if (raw.signalling && !wide) refuse(what, "a signalling NaN"); + return wide ? raw.value : Math.fround(raw.value); + } if (typeof raw === "number") return wide ? raw : Math.fround(raw); if (typeof raw === "bigint") return wide ? Number(raw) : bigintToFloat32(raw); return refuse(what, kindOf(raw)); @@ -187,17 +191,21 @@ type SlotsSpec = Readonly>, owner: string, low: number): number[] { +// 0 to MAX_SLOT_KEY, and a key outside its range is refused at definition rather than allocated for. +function slotKeys(spec: Readonly>, owner: string, low: number, high: number): number[] { return Object.keys(spec) .map((text) => { if (!/^-?(0|[1-9][0-9]*)$/u.test(text) || (low >= 0 && text.startsWith("-"))) { throw new Error(`${owner} declares the key "${text}", which is not ${low < 0 ? "an" : "a non-negative"} integer.`); } const key = BigInt(text); - if (key < BigInt(low) || key > BigInt(INT32_MAX)) { - throw new Error(`${owner} declares the key ${text}, which is outside Int32.`); + if (key < BigInt(low) || key > BigInt(high)) { + throw new Error(`${owner} declares the key ${text}, which is ${high === INT32_MAX ? "outside Int32" : `above the slot key limit ${high}`}.`); } return Number(key); }) @@ -217,7 +225,7 @@ function pascalCase(name: string): string { // are skipped, and a slot missing from the payload takes its declared default or is refused. A nil in a slot that // declares a default also takes the default (cultcache-py's rule). export function slots(spec: SlotsSpec, typeName?: string): SlotsCodec { - const keys = slotKeys(spec, "A slot record", 0); + const keys = slotKeys(spec, "A slot record", 0, MAX_SLOT_KEY); if (keys.length === 0) { throw new Error("A slot record declares no slots."); } @@ -289,7 +297,7 @@ export function union( arms: Readonly>>, typeName?: string, ): Codec { - const keys = slotKeys(arms, `Union ${name}`, INT32_MIN); + const keys = slotKeys(arms, `Union ${name}`, INT32_MIN, INT32_MAX); if (keys.length === 0) { throw new Error(`Union ${name} declares no arms.`); } diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index d9102237..12bf572a 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -4,7 +4,8 @@ // outside the safe range. A float decodes to a WireFloat, whatever its value. export class WireFloat { - constructor(readonly value: number, readonly wide: boolean) {} + // signalling: a float32 NaN whose quiet bit is clear. The JS number it becomes has lost that payload. + constructor(readonly value: number, readonly wide: boolean, readonly signalling = false) {} } const SAFE_MIN = BigInt(Number.MIN_SAFE_INTEGER); @@ -104,7 +105,11 @@ export function readWire(payload: Uint8Array): unknown { case 0xc4: return binary(view.getUint8(take(1))); case 0xc5: return binary(view.getUint16(take(2))); case 0xc6: return binary(view.getUint32(take(4))); - case 0xca: return new WireFloat(view.getFloat32(take(4)), false); + case 0xca: { + const at = take(4); + const bits = view.getUint32(at); + return new WireFloat(view.getFloat32(at), false, (bits & 0x7f800000) === 0x7f800000 && (bits & 0x007fffff) !== 0 && (bits & 0x00400000) === 0); + } case 0xcb: return new WireFloat(view.getFloat64(take(8)), true); case 0xcc: return view.getUint8(take(1)); case 0xcd: return view.getUint16(take(2)); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 6b31c902..c0f9524e 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -9,7 +9,7 @@ import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; import { codecs } from "../src"; -import type { Codec } from "../src/codec"; +import { MAX_SLOT_KEY, type Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; import { defineDocumentType } from "../src/document"; @@ -546,15 +546,27 @@ const isNaNBits = (bitsHex: string) => { return ((bits >> 52n) & 0x7ffn) === 0x7ffn && (bits & ((1n << 52n) - 1n)) !== 0n; }; +const isSignallingFloat32 = (payload: Uint8Array) => { + // payload is a one-slot record [float32]: 91 ca b0 b1 b2 b3 + if (payload.length !== 6 || payload[1] !== 0xca) return false; + const bits = new DataView(payload.buffer, payload.byteOffset).getUint32(2); + return (bits & 0x7f800000) === 0x7f800000 && (bits & 0x007fffff) !== 0 && (bits & 0x00400000) === 0; +}; + test("numeric leaves read and refuse exactly what the C# reference does", async () => { const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "reads.msgpack")))) as [string, Uint8Array, boolean, Uint8Array][]; - assert.equal(rows.length, 156); + assert.equal(rows.length, 160); let accepted = 0; let refused = 0; for (const [leaf, payload, wasAccepted, bits] of rows) { const label = `${leaf} ${hex(payload)}`; const record = slots<{ v: unknown }>({ 0: ["v", numericLeaves[leaf]] }); - if (wasAccepted) { + if (wasAccepted && leaf === "float32" && isSignallingFloat32(payload)) { + // The one reading that differs from the reference: it reads a float32 signalling NaN, this runtime refuses it + // (float32-nan.msgpack shows the reference keeps its bits, which a JS number cannot). + assert.throws(() => record.decode(readWire(payload)), (error: Error) => !error.message.includes(hex(payload).slice(2)), label); + refused += 1; + } else if (wasAccepted) { const read = record.decode(readWire(payload)).v as number | bigint; const wanted = hex(bits); if (leaf.startsWith("float")) { @@ -573,6 +585,25 @@ test("numeric leaves read and refuse exactly what the C# reference does", async assert.equal(accepted + refused, rows.length); }); +// float32-nan.msgpack: [payload, bytes the reference writes after reading it] for a float32 slot holding an infinity or a NaN. +// The reference keeps every payload bit; a JS number cannot keep a signalling NaN's, so that read is refused. +test("a float32 NaN round-trips with its bits, and a signalling one is refused on read", async () => { + const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "float32-nan.msgpack")))) as [Uint8Array, Uint8Array][]; + assert.equal(rows.length, 6); + const record = slots<{ v: number }>({ 0: ["v", float32] }); + let refused = 0; + for (const [payload, rewritten] of rows) { + if (isSignallingFloat32(payload)) { + assert.equal(hex(rewritten), hex(payload), "the reference keeps the signalling bits"); + assert.throws(() => record.decode(readWire(payload)), { message: "v (slot 0): expected a float32, found a signalling NaN" }, hex(payload)); + refused += 1; + } else { + assert.equal(hex(writeWire(record.encode(record.decode(readWire(payload))))), hex(rewritten), hex(payload)); + } + } + assert.equal(refused, 3); +}); + test("numeric leaves write the bytes the C# reference writes, and 64-bit values lose nothing", async () => { const rows = readWire(new Uint8Array(await readFile(join(numericVectors, "writes.msgpack")))) as [string, string, Uint8Array][]; assert.equal(rows.length, 46); @@ -673,11 +704,24 @@ test("arm keys span Int32 and are listed ascending; slot keys are non-negative I assert.throws(() => keys.decode(readWire(fromHex("920590"))), { message: "Union U: unknown arm key 5 (declared keys: -2147483648, -1, 0, 2147483647)." }); assert.throws(() => union<{ kind: "A" }>("U", { [-2147483649]: ["A", unit] }), { message: "Union U declares the key -2147483649, which is outside Int32." }); assert.throws(() => union<{ kind: "A" }>("U", { 99999999999999999999: ["A", unit] } as never), { message: "Union U declares the key 100000000000000000000, which is outside Int32." }); - assert.doesNotThrow(() => slots<{ a: string }>({ 2147483647: ["a", string] })); - assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is outside Int32." }); + assert.throws(() => slots<{ a: string }>({ 2147483647: ["a", string] }), { message: "A slot record declares the key 2147483647, which is above the slot key limit 65535." }); + assert.throws(() => slots<{ a: string }>({ 2147483648: ["a", string] }), { message: "A slot record declares the key 2147483648, which is above the slot key limit 65535." }); assert.throws(() => slots<{ a: string }>({ [-1]: ["a", string] }), { message: 'A slot record declares the key "-1", which is not a non-negative integer.' }); }); +// A slot record is a dense array of key+1 slots, as the reference writes it, so the key is capped: the largest accepted +// key writes a 65536-slot payload, one more is refused at definition. +test("a slot key is capped at 65535: the cap is the largest slot record, one past it is refused at definition", () => { + assert.equal(MAX_SLOT_KEY, 65535); + const edge = slots<{ a: string }>({ 65535: ["a", string] }); + const written = edge.encode({ a: "x" }) as unknown[]; + assert.equal(written.length, 65536); + assert.equal(written[65535], "x"); + assert.equal(written.filter((slot) => slot === null).length, 65535); + assert.deepEqual(edge.decode(readWire(writeWire(written))), { a: "x" }); + assert.throws(() => slots<{ a: string }>({ 65536: ["a", string] }), { message: "A slot record declares the key 65536, which is above the slot key limit 65535." }); +}); + test("the mixed-record vector is the bytes the reference writes for a record of every leaf kind", async () => { const mixed = slots<{ id: string; i: number; l: number | bigint; f: number; d: number }>({ 0: ["id", string], 1: ["i", int32], 2: ["l", int64], 3: ["f", float32], 4: ["d", float64], diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 36a5ae88..9c77e586 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -8,6 +8,7 @@ import { type CacheBackingStore, type CultCacheEnvelope, type CultCacheDocumentValue, + resolveDocumentFormatter, } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, @@ -2637,7 +2638,7 @@ export function cultMeshDocumentFromStore( if (!record) { throw new Error(`CultMesh store document '${documentId}' did not contain schema ${resolvedSchemaId}.`); } - return parseCultMeshDocumentValue(schemaOrDefinition, decode(record.payload)); + return parseCultMeshDocumentValue(schemaOrDefinition, decodeCultMeshPayload(schemaOrDefinition, record.payload)); }; return cultMeshDocument( @@ -2750,6 +2751,7 @@ export function cultMeshDocumentFromPeerSnapshot( parseCultMeshDocumentValue( schemaOrDefinition, decodeCultNetRawDocumentPayload( + schemaOrDefinition, await requestCultNetRawSnapshotDocument( peer, resolvedSchemaId, @@ -2761,7 +2763,7 @@ export function cultMeshDocumentFromPeerSnapshot( ? undefined : record => { try { - schemaOrDefinition.schema.parse(decodeCultNetRawDocumentPayload(record)); + schemaOrDefinition.schema.parse(decodeCultNetRawDocumentPayload(schemaOrDefinition, record)); return true; } catch { return false; @@ -4106,7 +4108,7 @@ export class CultMeshNode { messageIdPrefix: options.messageIdPrefix ?? key, accepts: record => { try { - definition.schema.parse(decodeCultNetRawDocumentPayload(record)); + definition.schema.parse(decodeCultNetRawDocumentPayload(definition, record)); return true; } catch { return false; @@ -6329,7 +6331,7 @@ export class CultMesh { const onMessage = (message: CultNetMessage): void => { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || message.messageId !== messageId || message.document.schemaId !== receiptBinding.definition.schemaId) return; try { - const decoded = decode(new Uint8Array(message.document.payload)); + const decoded = decodeCultMeshPayload(receiptBinding.definition, new Uint8Array(message.document.payload)); cleanup(); resolve(receiptBinding.definition.schema.parse(decoded) as CultCacheDocumentValue); } catch (error) { cleanup(); reject(error); } @@ -6638,7 +6640,7 @@ function cultMeshEnvelopeParsesAsDefinition( definition: AnyCultCacheDocumentDefinition, ): boolean { try { - definition.schema.parse(decode(envelope.payload)); + definition.schema.parse(decodeCultMeshPayload(definition, envelope.payload)); return true; } catch { return false; @@ -6774,11 +6776,22 @@ async function resolveCultNetPeer( : peerOrProvider; } -function decodeCultNetRawDocumentPayload(document: CultNetRawDocumentRecord): unknown { +// A document payload is read the way its definition's cache reads it (a codec or formatter document is not a plain +// msgpack map); a payload with no definition is plain msgpack. +function decodeCultMeshPayload(schemaOrDefinition: string | AnyCultCacheDocumentDefinition, payload: Uint8Array): unknown { + return typeof schemaOrDefinition === "string" + ? decode(payload) + : resolveDocumentFormatter(schemaOrDefinition).decode(payload); +} + +function decodeCultNetRawDocumentPayload( + schemaOrDefinition: string | AnyCultCacheDocumentDefinition, + document: CultNetRawDocumentRecord, +): unknown { if (document.payloadEncoding !== "messagepack") { throw new Error(`Unsupported CultNet raw document payload encoding ${document.payloadEncoding}.`); } - return decode(toUint8Array(document.payload)); + return decodeCultMeshPayload(schemaOrDefinition, toUint8Array(document.payload)); } function parseCultMeshDocumentValue( diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts new file mode 100644 index 00000000..0514cb99 --- /dev/null +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { z } from "zod"; +import { codecs, defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + defineCultNetDocumentBinding, + type CultNetPeer, +} from "cultnet-ts"; +import { CultMesh, cultMeshDocumentFromSingleFile } from "../src/index"; + +// A document written in the reference's positional shape, and a plain map document beside it. Every mesh reader of a +// payload reads it the way the cache does, so both read through the mesh. +const codecNote = defineDocumentType({ + type: "cultmesh.codec_note", + schemaId: "cultmesh.codec_note.v0", + schema: z.object({ noteId: z.string(), body: z.string() }), + name: "noteId", + codec: codecs.slots<{ noteId: string; body: string }>({ + 0: ["noteId", codecs.string], + 1: ["body", codecs.string], + }, "Tests.CodecNote"), +}); + +const codecReceipt = defineDocumentType({ + type: "cultmesh.codec_receipt", + schemaId: "cultmesh.codec_receipt.v0", + schema: z.object({ commandId: z.string(), body: z.string() }), + name: "commandId", + codec: codecs.slots<{ commandId: string; body: string }>({ + 0: ["commandId", codecs.string], + 1: ["body", codecs.string], + }, "Tests.CodecReceipt"), +}); + +const plainNote = defineDocumentType({ + type: "cultmesh.plain_note", + schemaId: "cultmesh.plain_note.v0", + schema: z.object({ noteId: z.string(), body: z.string() }), + name: "noteId", +}); + +test("the mesh reads a stored codec document as the cache does, and a plain document unchanged", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-store-")); + const codecPath = join(dir, "codec.ccmp"); + const plainPath = join(dir, "plain.ccmp"); + const codecNode = await CultMesh.startNode(codecPath, { documents: [codecNote] }); + await codecNode.put(codecNote, "n1", { noteId: "n1", body: "positional" }); + const plainNode = await CultMesh.startNode(plainPath, { documents: [plainNote] }); + await plainNode.put(plainNote, "n2", { noteId: "n2", body: "map" }); + + assert.deepEqual(await cultMeshDocumentFromSingleFile(codecPath, codecNote).latest(), { noteId: "n1", body: "positional" }); + assert.deepEqual(await cultMeshDocumentFromSingleFile(plainPath, plainNote).latest(), { noteId: "n2", body: "map" }); +}); + +test("the mesh syncs a codec document from a peer snapshot and returns a codec receipt", async () => { + const connectionId = 0x10203060; + const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-codec-sync-")); + const source = await CultMesh.startNode(join(dir, "source.ccmp"), { documents: [codecNote] }); + await source.put(codecNote, "n1", { noteId: "n1", body: "synced positional" }); + const target = await CultMesh.startNode(join(dir, "target.ccmp"), { documents: [codecNote] }); + const commandBinding = defineCultNetDocumentBinding({ definition: codecNote }); + const receiptBinding = defineCultNetDocumentBinding({ definition: codecReceipt }); + const server = CultMesh.createRudpDocumentServer("cultmesh-ts-codec-server", connectionId, { + documents: new CultNetDocumentRegistry([commandBinding, receiptBinding]), + getCache: () => source.cache, + bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, + onDocumentPutRaw: async (document) => ({ + binding: receiptBinding, + recordKey: document.recordKey, + value: { commandId: document.recordKey, body: "applied" }, + }), + }); + let peer: CultNetPeer | undefined; + try { + await server.start(); + peer = await CultMesh.createRudpPeer("cultmesh-ts-codec-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000, + }); + // The accepts probe reads the snapshot's payload as the codec: a plain decode would reject every codec snapshot. + const synced = await target.syncDocumentFromPeerSnapshot(peer, codecNote, "n1", { timeoutMs: 1_000 }); + assert.deepEqual(synced, { noteId: "n1", body: "synced positional" }); + assert.deepEqual(target.getRequired(codecNote, "n1"), { noteId: "n1", body: "synced positional" }); + + const receipt = await CultMesh.publishRudpDocumentAndWaitForReceipt( + "cultmesh-ts-codec-receipt", connectionId, `rudp://127.0.0.1:${server.bind.port}`, + commandBinding, "n2", { noteId: "n2", body: "command" }, receiptBinding, + { messageId: "n2", receiptTimeoutMs: 1_000, resendPollMs: 5 }, + ); + assert.deepEqual(receipt, { commandId: "n2", body: "applied" }); + } finally { + peer?.close(); + server.close(); + } +}); diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index 912fdb56..75b078d7 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -74,6 +74,8 @@ namespace GameCult.Caching.Tests "cf0020000000000001", "cfffffffffffffffff", "cf8000000000000000", "d30020000000000001", "d3fffffffffffffff0", // integers beyond 2^53 that a float32 slot must round once: past a tie, on a tie to even, on a tie to odd, an odd value just past exact, negative "d31000001000000001", "cf8000008000000001", "cf8000008000000000", "cf8000018000000000", "cf8000010000000001", "cf8000010000000000", "d3efffffe800000001", "d3effffff000000001", + // an integer a float32 cannot hold that a reader holds as a plain number: 2^24 + 1 rounds to 2^24 + "ce01000001", "ca3fc00000", "cb3ff8000000000000", "cb401c000000000000", "cb7ff8000000000000", "ca7f800000", "ca7f800001", "cb7e37e43c8800759c", "cb47efffffe0000000", "cb3fb999999999999a", "ca3dcccccd", "cb8000000000000000", "c0", "c3", "a161", "c40100", "90", "80", @@ -174,11 +176,46 @@ namespace GameCult.Caching.Tests // mixed-record.msgpack: the reference's bytes for Mixed { "n1", 7, 8, 1.5f, 2.5 }, which other runtimes must write identically. private static byte[] MixedRecord() => MessagePackSerializer.Serialize(new Mixed { Id = "n1", I = 7, L = 8, F = 1.5f, D = 2.5 }, Options); + // float32-nan.msgpack: an array of [payload (bin), bytes (bin)]: a one-slot float32 record holding an infinity or a NaN, + // and the bytes the reference writes after reading it. A signalling NaN (quiet bit clear) comes back unchanged. + private static readonly string[] Float32NanValues = { "ca7f800000", "ca7f800001", "ca7fa00000", "ca7fc00001", "ca7fe00000", "caff800001" }; + + private static byte[] Float32NanVector() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(Float32NanValues.Length); + foreach (var value in Float32NanValues) + { + var payload = Payload(value); + writer.WriteArrayHeader(2); + writer.Write(payload); + writer.Write(MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(payload, Options), Options)); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + // ---- tests ---- [Test] public void ReadsVectorMatches() => Check("reads.msgpack", ReadsVector()); + [Test] + public void Float32NanVectorMatches() => Check("float32-nan.msgpack", Float32NanVector()); + + [Test] + public void Float32SignallingNanIsReadAndWrittenBitForBit() + { + foreach (var value in new[] { "ca7f800001", "ca7fa00000", "caff800001" }) + { + var payload = Payload(value); + var again = MessagePackSerializer.Serialize(MessagePackSerializer.Deserialize(payload, Options), Options); + Assert.That(Convert.ToHexString(again), Is.EqualTo(Convert.ToHexString(payload)), value); + } + } + [Test] public void WritesVectorMatches() => Check("writes.msgpack", WritesVector());