diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index 54e75700..43fc41cd 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -1,5 +1,6 @@ import { decode } from "@msgpack/msgpack"; +import { checkPayloadDepth } from "./payload-depth"; import { STORE_FORMAT_VERSION, isV1Snapshot, requireV1RecordSlots } from "./store-format"; export interface CultCacheInspection { @@ -288,6 +289,7 @@ function decodeLegacyEnvelope(value: unknown): InspectedRecord { function previewPayload(payload: Uint8Array): Pick { try { + checkPayloadDepth(payload); return { payloadPreview: toJsonSafe(decode(payload)) }; } catch (error) { return { diff --git a/packages/cultcache-ts/src/document-formatter.ts b/packages/cultcache-ts/src/document-formatter.ts index 54dee1ed..799b2558 100644 --- a/packages/cultcache-ts/src/document-formatter.ts +++ b/packages/cultcache-ts/src/document-formatter.ts @@ -1,5 +1,6 @@ import { decode, encode } from "@msgpack/msgpack"; +import { MAX_PAYLOAD_DEPTH } from "./payload-depth"; import type { SlotsCodec } from "./codec"; import type { AnyCultCacheDocumentDefinition, CultCacheDocumentFormatter } from "./types"; import { readWire, writeWire } from "./wire"; @@ -16,7 +17,8 @@ export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinit } const plainDocumentFormatter: CultCacheDocumentFormatter = { - encode: (value: unknown) => encode(value), + // The encoder's own depth refusal (100 by default) is set above the limit, so the payload walk decides. + encode: (value: unknown) => encode(value, { maxDepth: MAX_PAYLOAD_DEPTH * 2 }), decode: (payload: Uint8Array) => decode(payload), }; diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 4c6ea040..03c2e4a0 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -1,4 +1,5 @@ import { resolveDocumentFormatter } from "./document-formatter"; +import { checkPayloadDepth } from "./payload-depth"; import type { AnyCultCacheDocumentDefinition, CultCacheSchemaCatalogMember, @@ -104,19 +105,25 @@ export function documentPayloadBytes(payload: DocumentPayload): Uint8Array { return payload as unknown as Uint8Array; } +export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; + // The one reader of a document payload: through the definition's formatter, codec or plain map, then its schema. A // reader holds the definition or it has no business reading the bytes, so there is no definition-less form. export function decodeDocumentPayload( definition: TDefinition, payload: DocumentPayload, ): CultCacheDocumentValue { - const decoded = resolveDocumentFormatter(definition).decode(documentPayloadBytes(payload)); + const bytes = documentPayloadBytes(payload); + checkPayloadDepth(bytes, `document type "${definition.type}" payload`); + const decoded = resolveDocumentFormatter(definition).decode(bytes); return definition.schema.parse(decoded) as CultCacheDocumentValue; } // The one writer: the definition's schema first, then its formatter, codec or plain map. export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition, value: unknown): DocumentPayload { - return documentPayloadFromBytes(resolveDocumentFormatter(definition).encode(definition.schema.parse(value))); + const bytes = resolveDocumentFormatter(definition).encode(definition.schema.parse(value)); + checkPayloadDepth(bytes, `document type "${definition.type}" value`); + return documentPayloadFromBytes(bytes); } // The schema members of a document: the ones it declares, else those its codec derives, else none. diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index c5dfbff1..8646d12e 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -10,6 +10,7 @@ export { documentPayloadFromBytes, encodeDocumentPayload, } from "./document"; +export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; export type { DocumentPayload } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; export type { diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts new file mode 100644 index 00000000..40ca7afe --- /dev/null +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -0,0 +1,70 @@ +// The one payload nesting rule in TypeScript, the twin of the C# reference's CultPayloadDepth.Check: every MessagePack +// array and map counts, whether a reader reads it or skips past it, and the outermost array is depth 1. Depth +// MAX_PAYLOAD_DEPTH is accepted and one container deeper is refused, on read and on write. +export const MAX_PAYLOAD_DEPTH = 256; + +export class CultCachePayloadDepthError extends Error { + constructor(readonly depth: number, readonly limit: number, subject: string) { + super(`${subject} nests ${depth} MessagePack containers deep; the limit is ${limit}.`); + this.name = "CultCachePayloadDepthError"; + } +} + +// Walks the headers of one MessagePack value iteratively (a count stack bounded by the limit, scalars skipped by length, +// nothing read) and throws at the first container past the limit. The message names the subject, the depth and the limit, +// never payload bytes. A payload that ends early is the reader's to refuse. +export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): void { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const remaining = new Float64Array(MAX_PAYLOAD_DEPTH); + let depth = 0; + let position = 0; + // The big-endian length that follows the code byte, when the payload is long enough to hold it. + const width = (size: 1 | 2 | 4): number | undefined => { + if (position + 1 + size > bytes.length) return undefined; + return size === 1 ? view.getUint8(position + 1) : size === 2 ? view.getUint16(position + 1) : view.getUint32(position + 1); + }; + while (position < bytes.length) { + const code = bytes[position]; + let header = 0; + let children = 0; + let skip = 0; + let container = false; + if (code <= 0x7f || code >= 0xe0 || code === 0xc0 || code === 0xc1 || code === 0xc2 || code === 0xc3) skip = 1; + else if (code <= 0x8f) { container = true; header = 1; children = 2 * (code & 0x0f); } + else if (code <= 0x9f) { container = true; header = 1; children = code & 0x0f; } + else if (code <= 0xbf) skip = 1 + (code & 0x1f); + else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return; skip = 2 + n; } + else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return; skip = 3 + n; } + else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return; skip = 5 + n; } + else if (code === 0xc7) { const n = width(1); if (n === undefined) return; skip = 3 + n; } + else if (code === 0xc8) { const n = width(2); if (n === undefined) return; skip = 4 + n; } + else if (code === 0xc9) { const n = width(4); if (n === undefined) return; skip = 6 + n; } + else if (code === 0xca || code === 0xce || code === 0xd2) skip = 5; + else if (code === 0xcb || code === 0xcf || code === 0xd3) skip = 9; + else if (code === 0xcc || code === 0xd0) skip = 2; + else if (code === 0xcd || code === 0xd1 || code === 0xd4) skip = 3; + else if (code === 0xd5) skip = 4; + else if (code === 0xd6) skip = 6; + else if (code === 0xd7) skip = 10; + else if (code === 0xd8) skip = 18; + else if (code === 0xdc) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = n; } + else if (code === 0xdd) { const n = width(4); if (n === undefined) return; container = true; header = 5; children = n; } + else if (code === 0xde) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = 2 * n; } + else { const n = width(4); if (n === undefined) return; container = true; header = 5; children = 2 * n; } + + if (container) { + if (depth === MAX_PAYLOAD_DEPTH) throw new CultCachePayloadDepthError(MAX_PAYLOAD_DEPTH + 1, MAX_PAYLOAD_DEPTH, subject); + position += header; + if (children > 0) { + remaining[depth++] = children; + continue; + } + } else { + position += skip; + } + + // One element is complete; close every container it was the last element of. + while (depth > 0 && --remaining[depth - 1] === 0) depth--; + if (depth === 0) return; + } +} diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index cdc6cf9e..d0f295d5 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -5,6 +5,7 @@ import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; import { documentPayloadBytes, documentPayloadFromBytes } from "./document"; +import { checkPayloadDepth } from "./payload-depth"; import { STORE_FORMAT_VERSION, isV1Snapshot, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, @@ -355,6 +356,7 @@ function resolveCatalogEntryForRecord( function inferSchemaVersionFromPayload(payload: Uint8Array): string | undefined { try { + checkPayloadDepth(payload); const decoded = decode(payload); if (Array.isArray(decoded) && typeof decoded[0] === "string") { return decoded[0]; diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts new file mode 100644 index 00000000..8392bfb7 --- /dev/null +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -0,0 +1,190 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { encode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import * as root from "../src"; +import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; +import { + CultCachePayloadDepthError, + MAX_PAYLOAD_DEPTH, + decodeDocumentPayload, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, +} from "../src/document"; +import { checkPayloadDepth } from "../src/payload-depth"; +import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; + +const L = MAX_PAYLOAD_DEPTH; +const CANARY = "CANARY-payload-depth-91d3"; + +const cat = (...parts: Uint8Array[]) => Uint8Array.from(parts.flatMap((part) => [...part])); +const bytes = (...values: number[]) => Uint8Array.from(values); +const repeat = (unit: Uint8Array, times: number) => { + const out = new Uint8Array(unit.length * times); + for (let index = 0; index < times; index += 1) out.set(unit, index * unit.length); + return out; +}; +// n containers nested in one another around a leaf; `open` is one container's header with one child (two for a map). +const chain = (open: Uint8Array, n: number, leaf: Uint8Array = bytes(0xc0)) => cat(repeat(open, n), leaf); +const arrays = (n: number, leaf?: Uint8Array) => chain(bytes(0x91), n, leaf); + +const refused = (payload: Uint8Array, label: string) => + assert.throws(() => checkPayloadDepth(payload), (error: unknown) => + error instanceof CultCachePayloadDepthError && error.depth === L + 1 && error.limit === L, label); + +test("the limit is 256: depth 256 passes and 257 is refused, for every array and map header width", () => { + assert.equal(L, 256); + const kinds: [string, Uint8Array][] = [ + ["fixarray", bytes(0x91)], + ["array16", bytes(0xdc, 0, 1)], + ["array32", bytes(0xdd, 0, 0, 0, 1)], + ["fixmap", bytes(0x81, 0x00)], + ["map16", bytes(0xde, 0, 1, 0x00)], + ["map32", bytes(0xdf, 0, 0, 0, 1, 0x00)], + ]; + for (const [name, open] of kinds) { + assert.doesNotThrow(() => checkPayloadDepth(chain(open, L)), `${name} at the limit`); + refused(chain(open, L + 1), `${name} past the limit`); + } +}); + +test("an empty map or array as the innermost container counts like any other", () => { + for (const empty of [bytes(0x90), bytes(0x80), bytes(0xdc, 0, 0), bytes(0xdd, 0, 0, 0, 0), bytes(0xde, 0, 0), bytes(0xdf, 0, 0, 0, 0)]) { + assert.doesNotThrow(() => checkPayloadDepth(arrays(L - 1, empty))); + refused(arrays(L, empty), "empty innermost past the limit"); + } + // [a chain of 255, a chain of 255]: the first sibling closes before the second opens, so the depth is 256, not 511. + const sibling = arrays(L - 1); + assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), sibling, sibling))); + refused(cat(bytes(0x92), sibling, arrays(L)), "the second sibling is one deeper"); + // A map's key and value both count as elements: {key: chain, key: chain} closes after four elements. + assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x82, 0x00), sibling, bytes(0x01), sibling))); + refused(cat(bytes(0x82, 0x00), sibling, bytes(0x01), arrays(L)), "the second map value is one deeper"); + refused(cat(bytes(0x82), arrays(L), sibling, bytes(0x01), sibling), "a map key one deeper"); +}); + +test("every scalar is skipped by its length, whatever bytes it holds", () => { + const filler = (count: number) => repeat(bytes(0x91), count); + const scalars: [string, Uint8Array][] = [ + ["nil", bytes(0xc0)], ["false", bytes(0xc2)], ["true", bytes(0xc3)], ["positive fixint", bytes(0x7f)], ["negative fixint", bytes(0xe0)], + ["uint8", bytes(0xcc, 0x91)], ["uint16", cat(bytes(0xcd), filler(2))], ["uint32", cat(bytes(0xce), filler(4))], ["uint64", cat(bytes(0xcf), filler(8))], + ["int8", bytes(0xd0, 0x91)], ["int16", cat(bytes(0xd1), filler(2))], ["int32", cat(bytes(0xd2), filler(4))], ["int64", cat(bytes(0xd3), filler(8))], + ["float32", cat(bytes(0xca), filler(4))], ["float64", cat(bytes(0xcb), filler(8))], + ["fixext1", cat(bytes(0xd4), filler(2))], ["fixext2", cat(bytes(0xd5), filler(3))], ["fixext4", cat(bytes(0xd6), filler(5))], + ["fixext8", cat(bytes(0xd7), filler(9))], ["fixext16", cat(bytes(0xd8), filler(17))], + ["fixstr", cat(bytes(0xa3), filler(3))], ["str8", cat(bytes(0xd9, 3), filler(3))], ["str16", cat(bytes(0xda, 0, 3), filler(3))], + ["str32", cat(bytes(0xdb, 0, 0, 0, 3), filler(3))], + ["bin8", cat(bytes(0xc4, 3), filler(3))], ["bin16", cat(bytes(0xc5, 0, 3), filler(3))], ["bin32", cat(bytes(0xc6, 0, 0, 0, 3), filler(3))], + ["ext8", cat(bytes(0xc7, 3, 0x7f), filler(3))], ["ext16", cat(bytes(0xc8, 0, 3, 0x7f), filler(3))], + ["ext32", cat(bytes(0xc9, 0, 0, 0, 3, 0x7f), filler(3))], + ["str16 of 300 bytes", cat(bytes(0xda, 0x01, 0x2c), filler(300))], + ]; + for (const [name, scalar] of scalars) { + assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), scalar, arrays(L - 1))), `${name} then a chain at the limit`); + refused(cat(bytes(0x92), scalar, arrays(L)), `${name} then a chain past it`); + assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), arrays(L - 1), scalar)), `${name} after a chain at the limit`); + } +}); + +test("a payload that ends early, is empty or carries trailing bytes is the reader's business, not the walk's", () => { + for (const early of [[], [0xd9], [0xda, 0], [0xdb, 0, 0, 0], [0xc4], [0xc5, 0], [0xc6, 0, 0, 0], [0xc7], [0xc8, 0], [0xc9, 0, 0, 0], [0xdc, 0], [0xdd, 0, 0, 0], [0xde, 0], [0xdf, 0, 0, 0]]) { + assert.doesNotThrow(() => checkPayloadDepth(bytes(...early)), early.map((value) => value.toString(16)).join(" ")); + } + assert.doesNotThrow(() => checkPayloadDepth(arrays(L - 1, bytes(0x91))), "a chain cut off before its leaf"); + assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), "bytes after the first complete value are not walked"); + assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0))); +}); + +test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { + refused(arrays(100_000), "100000 nested arrays"); + refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps"); +}); + +test("a refusal names the subject, the depth and the limit and never a byte of the payload", () => { + const payload = chain(bytes(0x91), L + 1, cat(bytes(0xd9, CANARY.length), new TextEncoder().encode(CANARY))); + assert.throws( + () => checkPayloadDepth(payload, 'document type "t.note" payload'), + (error: unknown) => error instanceof CultCachePayloadDepthError + && error.name === "CultCachePayloadDepthError" + && error.message === 'document type "t.note" payload nests 257 MessagePack containers deep; the limit is 256.', + ); + const canaryPayload = payload; + assert.throws(() => checkPayloadDepth(canaryPayload), (error: Error) => !error.message.includes(CANARY) && !JSON.stringify(error).includes(CANARY)); +}); + +const note = z.object({ id: z.string(), count: z.number() }); +const formatterCalls: string[] = []; +const guarded = defineDocumentType({ + type: "t.guarded", + schemaId: "t.guarded.v0", + schema: note, + name: "id", + formatter: { + encode: () => (formatterCalls.push("encode"), arrays(L + 1)), + decode: () => (formatterCalls.push("decode"), { id: "a", count: 1 }), + }, +}); + +test("the owner refuses a payload past the limit before its formatter reads it, and a formatter's bytes before they are wrapped", () => { + assert.throws(() => decodeDocumentPayload(guarded, documentPayloadFromBytes(arrays(L + 1))), CultCachePayloadDepthError); + assert.deepEqual(formatterCalls, [], "the formatter never saw the deep payload"); + assert.deepEqual(decodeDocumentPayload(guarded, documentPayloadFromBytes(arrays(L))), { id: "a", count: 1 }); + + assert.throws(() => encodeDocumentPayload(guarded, { id: "a", count: 1 }), (error: Error) => + error instanceof CultCachePayloadDepthError && error.message === 'document type "t.guarded" value nests 257 MessagePack containers deep; the limit is 256.'); +}); + +test("a document without a codec or formatter writes a value L deep and refuses L+1 by the walk", () => { + const anything = defineDocumentType({ type: "t.any", schemaId: "t.any.v0", schema: z.any(), name: "id" }); + const nest = (n: number): unknown => (n === 1 ? [] : [nest(n - 1)]); + const atLimit = encodeDocumentPayload(anything, nest(L)); + assert.deepEqual(documentPayloadBytes(atLimit), encode(nest(L), { maxDepth: 1000 })); + assert.deepEqual(decodeDocumentPayload(anything, atLimit), nest(L)); + assert.throws(() => encodeDocumentPayload(anything, nest(L + 1)), (error: unknown) => + error instanceof CultCachePayloadDepthError && error.depth === L + 1 && error.limit === L); + assert.throws(() => decodeDocumentPayload(anything, documentPayloadFromBytes(arrays(L + 1))), CultCachePayloadDepthError); +}); + +test("the limit and its error are exported from the root, and the walk itself is not", () => { + assert.equal(root.MAX_PAYLOAD_DEPTH, 256); + assert.equal(root.CultCachePayloadDepthError, CultCachePayloadDepthError); + assert.equal("checkPayloadDepth" in root, false); +}); + +// A v1 store whose one record carries `payload` as bin and no catalog entry, so the record's schema is inferred from the +// payload's first element. +async function storeWith(payload: Uint8Array): Promise { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-payload-depth-")), "deep.cc"); + await writeFile(file, encode(["cultcache.store.v1", [], [["record-1", "sha256:stale", "2026-06-25T12:00:00Z", payload]]])); + return file; +} +const stamped = (containersInsideTheStamp: number) => + cat(bytes(0x92), encode("tests.schema_stamped_entry.v1"), arrays(containersInsideTheStamp)); + +test("the inspector previews a payload at the limit and reports one past it, or 100000 deep, as a decode error", async () => { + const atLimit = await storeWith(stamped(L - 1)); + const previewed = inspectCultCacheBytes(atLimit, await readFile(atLimit)).records[0]!; + assert.equal(previewed.payloadDecodeError, undefined); + + for (const inside of [L, 100_000]) { + const file = await storeWith(stamped(inside)); + const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!; + assert.equal(record.payloadDecodeError, "document payload nests 257 MessagePack containers deep; the limit is 256."); + } +}); + +test("the single-file store infers a legacy record's schema from a payload at the limit and from none past it", async () => { + const atLimit = await storeWith(stamped(L - 1)); + const [read] = await new SingleFileMessagePackBackingStore(atLimit).pullAll(); + assert.equal(read.catalogEntry?.schemaVersion, "tests.schema_stamped_entry.v1"); + + const past = await storeWith(stamped(L)); + await assert.rejects(() => new SingleFileMessagePackBackingStore(past).pullAll(), /missing schema id/u); +}); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index b025f113..575da1d8 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -12,7 +12,15 @@ import { codecs } from "../src"; import { MAX_SLOT_KEY, type Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; -import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; +import { + CultCachePayloadDepthError, + MAX_PAYLOAD_DEPTH, + decodeDocumentPayload, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, +} from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; @@ -39,9 +47,10 @@ type Node = | { kind: "Box"; id: string; children: (Node | null)[] } | { kind: "Text"; id: string; content: string } | { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] } - | { kind: "Image"; id: string; uri: string; alt: string }; + | { kind: "Image"; id: string; uri: string; alt: string } + | { kind: "Wrap"; inner: Node | null }; interface Composition { name: string; body: Node | null } -type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" }; +type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" }; type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; interface Surface { @@ -62,11 +71,14 @@ const node: Codec = union("Node", { 0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))], })], 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], + 6: ["Wrap", slots<{ inner: Node | null }>({ 0: ["inner", lazy(() => node)] })], }, `${CS}Node`); const outcome = union("Outcome", { 0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })], 1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", int32] })], 2: ["Denied", unit], + [-2147483648]: ["Floor", unit], + 2147483647: ["Ceiling", unit], }, `${CS}Outcome`); const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] }); const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] }); @@ -99,6 +111,7 @@ const nodeSchema: z.ZodType = z.lazy(() => slots: z.array(nodeSchema), }), z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }), + z.object({ kind: z.literal("Wrap"), inner: nodeSchema }), ]), ])); const surfaceSchema = z.object({ @@ -111,6 +124,8 @@ const surfaceSchema = z.object({ z.object({ kind: z.literal("Pending"), message: z.string() }), z.object({ kind: z.literal("Completed"), sourceVersion: z.number() }), z.object({ kind: z.literal("Denied") }), + z.object({ kind: z.literal("Floor") }), + z.object({ kind: z.literal("Ceiling") }), ]), ])), part: z.union([z.null(), z.discriminatedUnion("kind", [ @@ -273,7 +288,7 @@ test("an arm that is not [key, slots] is refused naming the union, and a non-int }); test("an arm key that is not a declared key is refused, with no fallback", () => { - assert.throws(() => node.decode([4, []]), /Union Node: unknown arm key 4 \(declared keys: 0, 1, 2, 5\)/u); + assert.throws(() => node.decode([4, []]), /Union Node: unknown arm key 4 \(declared keys: 0, 1, 2, 5, 6\)/u); assert.throws(() => outcome.decode([3, []]), /Union Outcome: unknown arm key 3/u); }); @@ -448,14 +463,14 @@ test("an arm key is an integer in any MessagePack integer encoding, and a float- assert.deepEqual(outcome.decode(readWire(fromHex(`92${key}9107`))), completed, key); } for (const key of ["ca3f800000", "cb3ff0000000000000"]) { - assert.throws(() => outcome.decode(readWire(fromHex(`92${key}9107`))), { message: "Union Outcome: the arm key is not an integer (declared keys: 0, 1, 2)." }, key); + assert.throws(() => outcome.decode(readWire(fromHex(`92${key}9107`))), { message: "Union Outcome: the arm key is not an integer (declared keys: -2147483648, 0, 1, 2, 2147483647)." }, key); } assert.throws(() => outcome.decode(readWire(fromHex("92cb800000000000000091a477616974"))), /^Error: Union Outcome: the arm key is not an integer/u, "-0.0 is a float, not key 0"); for (const key of ["ce80000000", "cf8000000000000000", "d30000000080000000"]) { - assert.throws(() => outcome.decode(readWire(fromHex(`92${key}9107`))), { message: "Union Outcome: the arm key is outside Int32 (declared keys: 0, 1, 2)." }, key); + assert.throws(() => outcome.decode(readWire(fromHex(`92${key}9107`))), { message: "Union Outcome: the arm key is outside Int32 (declared keys: -2147483648, 0, 1, 2, 2147483647)." }, key); } - assert.throws(() => outcome.decode(readWire(fromHex("92d2800000009107"))), { message: "Union Outcome: unknown arm key -2147483648 (declared keys: 0, 1, 2)." }); - assert.throws(() => outcome.decode(readWire(fromHex("92ff9107"))), { message: "Union Outcome: unknown arm key -1 (declared keys: 0, 1, 2)." }); + assert.throws(() => outcome.decode(readWire(fromHex("92d2800000019107"))), { message: "Union Outcome: unknown arm key -2147483647 (declared keys: -2147483648, 0, 1, 2, 2147483647)." }); + assert.throws(() => outcome.decode(readWire(fromHex("92ff9107"))), { message: "Union Outcome: unknown arm key -1 (declared keys: -2147483648, 0, 1, 2, 2147483647)." }); }); test("a union refuses a declared arm key outside Int32", () => { @@ -787,3 +802,85 @@ test("a bytes slot reads a MessagePack str as its bytes and writes bin, and refu assert.throws(() => record.decode(readWire(fromHex("91a2c328"))), { message: "b (slot 0): expected bytes, found a string that is not valid UTF-8" }); assert.throws(() => record.decode([5]), { message: "b (slot 0): expected bytes, found number" }); }); + +// ---- the bounds vectors: arm keys at the ends of the Int32 domain, nil arm slots, payload nesting ---- + +const wrapChain = (wraps: number, leaf: Node): Node => { + let node: Node = leaf; + for (let index = 0; index < wraps; index += 1) node = { kind: "Wrap", inner: node }; + return node; +}; +// A Wrap adds 2 containers, a Box with no children 3, a Text leaf 2, and the outermost array is 1 (the C# test's arithmetic). +const AT_LIMIT_WRAPS = (MAX_PAYLOAD_DEPTH - 1 - 3) / 2; +const PAST_LIMIT_WRAPS = (MAX_PAYLOAD_DEPTH + 1 - 1 - 2) / 2; +const atLimitValue: Surface = { ...golden, root: wrapChain(AT_LIMIT_WRAPS, { kind: "Box", id: "leaf", children: [] }) }; +const pastLimitValue: Surface = { ...golden, root: wrapChain(PAST_LIMIT_WRAPS, { kind: "Text", id: "leaf", content: CANARY }) }; + +async function boundsEntries() { + const raw = decode(await read("bounds-outcomes.msgpack")) as unknown[][]; + const [limit, ...rest] = raw; + return { limit, entries: rest.map(([file, outcome, unionName, key, detail]) => ({ file: file as string, outcome: outcome as string, unionName: unionName as string, key: key as number | null, detail: detail as string })) }; +} + +test("every bounds vector is listed and holds: accepted files decode and re-encode, refused files are refused naming the union or the depth", async () => { + const { limit, entries } = await boundsEntries(); + assert.deepEqual(limit, ["limit", MAX_PAYLOAD_DEPTH]); + assert.equal(entries.length, 12); + const document = surfaceDocument(); + const counts = { accepted: 0, refused: 0, readOnly: 0 }; + + for (const { file, outcome: expected, unionName, key, detail } of entries) { + const payload = documentPayloadFromBytes(await read(file)); + if (expected === "accepted") { + counts.accepted += 1; + const value = decodeDocumentPayload(document, payload); + assert.equal(hex(documentPayloadBytes(encodeDocumentPayload(document, value))), hex(await read(detail)), `${file}: re-encodes to ${detail}`); + } else if (expected === "read-only") { + counts.readOnly += 1; + assert.deepEqual(decodeDocumentPayload(document, payload), golden, `${file}: the extra slots are read past`); + } else { + counts.refused += 1; + assert.throws( + () => decodeDocumentPayload(document, payload), + (error: Error) => { + assert.ok(!error.message.includes(CANARY), `${file}: the refusal echoed the payload`); + if (detail === "depth") { + return error instanceof CultCachePayloadDepthError && error.depth === MAX_PAYLOAD_DEPTH + 1 && error.limit === MAX_PAYLOAD_DEPTH + && error.message.includes(`"${document.type}"`) && error.message.includes("257") && error.message.includes("256"); + } + assert.ok(!(error instanceof CultCachePayloadDepthError), file); + const said = detail === "key-out-of-range" ? "outside Int32" : detail === "key-not-integer" ? "not an integer" : "expected [key, slots]"; + return error.message.includes(`Union ${unionName}:`) && error.message.includes(said); + }, + file, + ); + } + } + assert.deepEqual(counts, { accepted: 3 + 1, refused: 4 + 1 + 2, readOnly: 1 }); +}); + +test("the constructed values write the committed at-limit payload byte for byte, and the past-limit value is refused and stored nowhere", async () => { + const document = surfaceDocument(); + assert.equal(hex(documentPayloadBytes(encodeDocumentPayload(document, atLimitValue))), hex(await read("depth-at-limit.msgpack"))); + assert.deepEqual(decodeDocumentPayload(document, documentPayloadFromBytes(await read("depth-at-limit.msgpack"))), atLimitValue); + + assert.throws( + () => encodeDocumentPayload(document, pastLimitValue), + (error: Error) => error instanceof CultCachePayloadDepthError && error.depth === 257 && !error.message.includes(CANARY), + ); + + const entries: CultCacheEnvelope[] = []; + const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build(); + await assert.rejects(() => cache.put(document, "past", pastLimitValue), CultCachePayloadDepthError); + assert.deepEqual(entries, [], "nothing reached the store"); + assert.equal(cache.get(document, "past"), undefined, "nothing is held in memory"); + await cache.put(document, "at", atLimitValue); + assert.equal(entries.length, 1); + assert.equal(hex(documentPayloadBytes(entries[0]!.payload)), hex(await read("depth-at-limit.msgpack"))); +}); + +test("a cache load refuses a store whose record is past the limit, naming the type and not the payload", async () => { + const document = surfaceDocument(); + const refusing = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(await read("depth-past-limit.msgpack"))])).build(); + await assert.rejects(() => refusing.pullAllBackingStores(), (error: Error) => error instanceof CultCachePayloadDepthError && !error.message.includes(CANARY)); +}); diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index eec87133..4843c09b 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -5,10 +5,14 @@ import { join } from "node:path"; import test from "node:test"; import { z } from "zod"; import { + CultCachePayloadDepthError, + MAX_PAYLOAD_DEPTH, + SingleFileMessagePackBackingStore, codecs, decodeDocumentPayload, defineDocumentType, documentPayloadBytes, + documentPayloadFromBytes, encodeDocumentPayload, } from "@gamecult/cultcache-ts"; import type { CacheBackingStore, CultCacheEnvelope } from "@gamecult/cultcache-ts"; @@ -208,3 +212,50 @@ test("a store read takes the record the catalog matches to the schema and the de const store: CacheBackingStore = { pullAll: async () => records, push: async () => undefined, delete: async () => undefined }; assert.deepEqual(await cultMeshDocumentFromStore(store, target).latest(), { n: 2 }); }); + +// A payload nested past the limit: MAX_PAYLOAD_DEPTH + 1 one-element arrays around a nil. +const pastLimitPayload = () => documentPayloadFromBytes(Uint8Array.from([...new Array(MAX_PAYLOAD_DEPTH + 1).fill(0x91), 0xc0])); + +test("a payload past the nesting limit is refused at the network doors: a RUDP put and a single-file read", async () => { + const connectionId = 0x10203080; + const plainBinding = defineCultNetDocumentBinding({ definition: plainNote }); + const registry = new CultNetDocumentRegistry([plainBinding]); + const errors: unknown[] = []; + const puts: CultMeshRudpDocumentPut[] = []; + const server = CultMesh.createRudpDocumentServer("cultmesh-ts-depth-server", connectionId, { + documents: registry, + bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, + onError: (error) => errors.push(error), + onDocumentPutRaw: async (document) => { puts.push(document); }, + }); + let peer: CultNetPeer | undefined; + try { + await server.start(); + peer = await CultMesh.createRudpPeer("cultmesh-ts-depth-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000, + }); + peer.send(registry.createRawDocumentPutMessageFromEnvelope("deep-1", { + key: "k", type: plainNote.type, schemaId: plainNote.schemaId, payload: pastLimitPayload(), storedAt: new Date().toISOString(), + })); + const startedAt = Date.now(); + while (errors.length === 0 && Date.now() - startedAt < 2_000) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(errors.length, 1); + assert.ok(errors[0] instanceof CultCachePayloadDepthError, "the put is refused by the payload owner"); + assert.deepEqual(puts, [], "the handler never saw the put"); + + // The server is still serving: an ordinary put goes through. + peer.send(registry.createRawDocumentPutMessage(plainBinding, "ok-1", "k2", { noteId: "k2", body: "fine" })); + while (puts.length === 0 && Date.now() - startedAt < 4_000) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(puts.length, 1); + } finally { + peer?.close(); + server.close(); + } + + const dir = await mkdtemp(join(tmpdir(), "cultmesh-ts-depth-store-")); + const path = join(dir, "deep.ccmp"); + await new SingleFileMessagePackBackingStore(path).push({ + key: "k", type: plainNote.type, schemaId: plainNote.schemaId, payload: pastLimitPayload(), storedAt: new Date().toISOString(), + }); + await assert.rejects(() => cultMeshDocumentFromSingleFile(path, plainNote).latest(), CultCachePayloadDepthError); +}); diff --git a/scripts/test-typescript-package-closure.mjs b/scripts/test-typescript-package-closure.mjs index 3eaf344a..58a40ae7 100644 --- a/scripts/test-typescript-package-closure.mjs +++ b/scripts/test-typescript-package-closure.mjs @@ -84,10 +84,19 @@ const mesh = require("cultmesh-ts"); assert.equal(typeof cache.defineDocumentType, "function"); // The document owner is also an entry of its own, which the browser client imports without the Node-only cache. assert.equal(documentEntry.decodeDocumentPayload, cache.decodeDocumentPayload); -assert.equal("resolveDocumentFormatter" in cache, false); -assert.equal("resolveDocumentFormatter" in documentEntry, false); -assert.equal("codecFormatter" in cache.codecs, false); -assert.equal("codecFormatter" in documentEntry, false); +// No entry exports a decoder or formatter that reads a payload around its schema: by shape, every packed export whose name +// says it decodes, reads a wire or formats is enumerated, and the only one allowed is the schema-checked owner. +const inspection = require("@gamecult/cultcache-ts/inspection"); +const decoderShaped = (entry) => Object.keys(entry).filter((name) => /decode|formatter|readwire/i.test(name)).sort(); +for (const [entryName, entry] of Object.entries({ root: cache, codecs: cache.codecs, document: documentEntry, inspection })) { + const found = decoderShaped(entry); + const allowed = entryName === "root" || entryName === "document" ? ["decodeDocumentPayload"] : []; + assert.deepEqual(found, allowed, "decoder-shaped exports of the " + entryName + " entry"); +} +assert.equal(typeof documentEntry.checkPayloadDepth, "undefined"); +assert.equal(documentEntry.MAX_PAYLOAD_DEPTH, 256); +assert.equal(cache.MAX_PAYLOAD_DEPTH, 256); +assert.equal(documentEntry.CultCachePayloadDepthError, cache.CultCachePayloadDepthError); assert.equal(typeof net.CultNetPeer, "function"); assert.equal(typeof mesh.CultMesh, "function"); assert.equal(typeof mesh.CultMeshProviderSession, "function");