From be968a71e614202f538d4efabec0db20df983018 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:08:24 +0000 Subject: [PATCH] Bytes after the value: a payload is one MessagePack value (TypeScript) decodeDocumentPayload refuses bytes after the first value by count, from the checkPayloadDepth walk's end, with the shared clause; readWire's own trailing check is deleted. Walks bytes-after-outcomes.msgpack on the plain, codec, store and putEnvelope paths. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 2 + packages/cultcache-ts/README.md | 6 +- packages/cultcache-ts/src/document.ts | 8 +- packages/cultcache-ts/src/payload-depth.ts | 28 +++--- packages/cultcache-ts/src/wire.ts | 5 +- .../cultcache-ts/test/payload-depth.test.ts | 10 +++ .../cultcache-ts/test/union-parity.test.ts | 89 ++++++++++++++++++- 7 files changed, 128 insertions(+), 20 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index e71821c8..e45dc14e 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,9 +20,11 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. +- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication), refuses bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse, and a user formatter's bytes are not walked. The refused set is unchanged, but the message was `@msgpack`'s `RangeError` ("Extra N of M byte(s)") for a plain document and the codec reader's "bytes after its value" for a codec document, so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Implied bump: 0.16.0 (pre-1.0 minor). ### Added +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (it was `void`); it still refuses only depth. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 2096d258..041e90ae 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -49,7 +49,9 @@ through `decodeDocumentPayload`, and when you need the bytes themselves (hashing write) take them with `documentPayloadBytes`, the sanctioned escape. - `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter, - codec or plain map and then its schema. The definition is required. + codec or plain map and then its schema. The definition is required. A payload is exactly one + MessagePack value: bytes after the first are refused by count, naming the type, and never walked or + echoed (a payload that ends early is the decoder's to refuse). - `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter, codec or plain map. - `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a @@ -327,7 +329,7 @@ If you do not supply one, `CultCacheTS` uses MessagePack plus Zod validation by The 256-container nesting bound (`MAX_PAYLOAD_DEPTH`, refused with `CultCachePayloadDepthError`) covers the MessagePack the library itself writes: the default format and a codec. A payload your formatter writes is opaque bytes in any format, so the -library does not walk it as MessagePack; its format and its bounds are your formatter's. +library does not walk it as MessagePack; its format, its bounds and whether it is one value are your formatter's. Zod is no longer mandatory. Any schema object with a `parse(...)` function will do, which means generated Ajv-backed JSON-schema contracts can feed the cache diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 10af96a6..3443451e 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -119,7 +119,13 @@ export function decodeDocumentPayload { const bytes = documentPayloadBytes(payload); - if (writesLibraryMessagePack(definition)) checkPayloadDepth(bytes, `document type "${definition.type}" payload`); + if (writesLibraryMessagePack(definition)) { + // A payload is exactly one MessagePack value: bytes after the first are refused by count, from the walk's end, before + // any formatter reads; the tail is never walked or echoed. A payload that ends early stays the decoder's to refuse. + const subject = `document type "${definition.type}" payload`; + const end = checkPayloadDepth(bytes, subject); + if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + } const decoded = resolveDocumentFormatter(definition).decode(bytes); return definition.schema.parse(decoded) as CultCacheDocumentValue; } diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts index 40ca7afe..efbc3a74 100644 --- a/packages/cultcache-ts/src/payload-depth.ts +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -12,8 +12,9 @@ export class CultCachePayloadDepthError extends Error { // Walks the headers of one MessagePack value iteratively (a count stack bounded by the limit, scalars skipped by length, // nothing read) and throws at the first container past the limit. The message names the subject, the depth and the limit, -// never payload bytes. A payload that ends early is the reader's to refuse. -export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): void { +// never payload bytes. Returns the offset where the first value ends, or undefined when the payload ends early (the +// reader's to refuse); what follows that offset is never walked, and whether it may exist is the caller's rule. +export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): number | undefined { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const remaining = new Float64Array(MAX_PAYLOAD_DEPTH); let depth = 0; @@ -33,12 +34,12 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code <= 0x8f) { container = true; header = 1; children = 2 * (code & 0x0f); } else if (code <= 0x9f) { container = true; header = 1; children = code & 0x0f; } else if (code <= 0xbf) skip = 1 + (code & 0x1f); - else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return; skip = 2 + n; } - else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return; skip = 5 + n; } - else if (code === 0xc7) { const n = width(1); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc8) { const n = width(2); if (n === undefined) return; skip = 4 + n; } - else if (code === 0xc9) { const n = width(4); if (n === undefined) return; skip = 6 + n; } + else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return undefined; skip = 2 + n; } + else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return undefined; skip = 5 + n; } + else if (code === 0xc7) { const n = width(1); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc8) { const n = width(2); if (n === undefined) return undefined; skip = 4 + n; } + else if (code === 0xc9) { const n = width(4); if (n === undefined) return undefined; skip = 6 + n; } else if (code === 0xca || code === 0xce || code === 0xd2) skip = 5; else if (code === 0xcb || code === 0xcf || code === 0xd3) skip = 9; else if (code === 0xcc || code === 0xd0) skip = 2; @@ -47,10 +48,10 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code === 0xd6) skip = 6; else if (code === 0xd7) skip = 10; else if (code === 0xd8) skip = 18; - else if (code === 0xdc) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = n; } - else if (code === 0xdd) { const n = width(4); if (n === undefined) return; container = true; header = 5; children = n; } - else if (code === 0xde) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = 2 * n; } - else { const n = width(4); if (n === undefined) return; container = true; header = 5; children = 2 * n; } + else if (code === 0xdc) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = n; } + else if (code === 0xdd) { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = n; } + else if (code === 0xde) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = 2 * n; } + else { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = 2 * n; } if (container) { if (depth === MAX_PAYLOAD_DEPTH) throw new CultCachePayloadDepthError(MAX_PAYLOAD_DEPTH + 1, MAX_PAYLOAD_DEPTH, subject); @@ -65,6 +66,7 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload // One element is complete; close every container it was the last element of. while (depth > 0 && --remaining[depth - 1] === 0) depth--; - if (depth === 0) return; + if (depth === 0) return position; } + return undefined; } diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index 12bf572a..d351c446 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -128,7 +128,6 @@ export function readWire(payload: Uint8Array): unknown { } } - const tree = next(); - if (offset !== payload.length) throw new Error("the MessagePack payload has bytes after its value."); - return tree; + // One value is read; the caller owns what follows it (decodeDocumentPayload refuses a payload with a tail). + return next(); } diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index 382f0672..e792accb 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -119,6 +119,16 @@ test("a payload that ends early, is empty or carries trailing bytes is the reade assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0))); }); +test("the walk returns where the first value ends, and undefined when the payload ends early", () => { + assert.equal(checkPayloadDepth(bytes(0xc0)), 1); + assert.equal(checkPayloadDepth(bytes(0xc0, 0xc1, 0xc1)), 1, "a top-level scalar ends at its own length; the tail is not judged"); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01, 0xa1, 0x61, 0xc0)), 5 - 1, "a container ends when its last child completes"); + assert.equal(checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), 1, "the tail is never walked"); + assert.equal(checkPayloadDepth(bytes()), undefined); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined); + assert.equal(checkPayloadDepth(bytes(0xd9)), undefined); +}); + test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { refused(arrays(100_000), "100000 nested arrays"); refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps"); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 46dcb42f..f71924cc 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -515,7 +515,7 @@ test("the wire reader keeps 64-bit integers whole, tells a float from an integer assert.throws(() => readWire(fromHex("dd")), /truncated/u); assert.throws(() => readWire(fromHex("ddffffffff")), /truncated/u, "a huge array header reserves nothing"); assert.throws(() => readWire(fromHex("a5616263")), /truncated/u); - assert.throws(() => readWire(fromHex("0102")), /bytes after its value/u); + assert.equal(readWire(fromHex("0102")), 1, "the reader reads one value; the document owner refuses what follows"); assert.throws(() => readWire(fromHex("81a161a162")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("c1")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("d4010a")), /map, extension or reserved/u); @@ -915,3 +915,90 @@ test("a cache load refuses a store whose record is past the limit, naming the ty const refusing = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(await read("depth-past-limit.msgpack"))])).build(); await assert.rejects(() => refusing.pullAllBackingStores(), (error: Error) => error instanceof CultCachePayloadDepthError && !error.message.includes(CANARY)); }); + +// ---- bytes-after vectors: a payload is exactly one MessagePack value (tests/vectors/union-parity/README.md) ---- + +// The C# reference writes the vectors: union-surface.msgpack followed by a tail the first value does not own. The one +// owner refuses by count, naming the type, from the walk's end; it never walks or echoes the tail. +async function bytesAfterEntries() { + const raw = decode(await read("bytes-after-outcomes.msgpack")) as unknown[][]; + return raw.map(([file, outcome, unionName, key, detail]) => ({ file: file as string, outcome: outcome as string, unionName: unionName as string, key: key as number | null, detail: detail as string })); +} + +// The C# vector writer's canary (BytesAfterVectorTests): it sits in the deep tail, so an echo of the tail would carry it. +const TAIL_CANARY = "CANARY-bytes-after-7d1e"; +const valueLength = async () => (await read("union-surface.msgpack")).length; +const plainDocument = () => defineDocumentType({ type: "tests.plain_surface", schema: z.unknown() }); +const codecDocument = () => defineDocumentType({ type: "tests.codec_surface", schema: surfaceSchema, codec: surfaceCodec }); + +async function assertBytesAfterRefused(file: string, tail: number, run: () => unknown | Promise, type: string) { + const clause = `${type}`; + await assert.rejects( + async () => run(), + (error: Error) => { + assert.ok(!(error instanceof CultCachePayloadDepthError), `${file}: the tail was walked`); + assert.ok(!error.message.includes(CANARY) && !error.message.includes(TAIL_CANARY), `${file}: the refusal echoed the tail`); + assert.ok(!error.message.includes("the limit is"), `${file}: the depth refusal`); + assert.ok(error.message.includes(clause), `${file}: names the type: ${error.message}`); + assert.ok(error.message.includes(`${tail} bytes after its first MessagePack value`), `${file}: names the count: ${error.message}`); + return true; + }, + file, + ); +} + +test("every bytes-after vector is refused by the one owner, by count, on the codec, plain and store paths", async () => { + const entries = await bytesAfterEntries(); + assert.equal(entries.length, 4); + const length = await valueLength(); + const surface = surfaceDocument(); + const plain = plainDocument(); + const codecOnly = codecDocument(); + const dir = await mkdtemp(join(tmpdir(), "bytes-after-")); + for (const { file, outcome, unionName, detail } of entries) { + assert.equal(outcome, "refused"); + assert.equal(unionName, "UnionSurface"); + assert.equal(detail, "bytes-after"); + const bytes = await read(file); + const tail = bytes.length - length; + assert.ok(tail > 0, file); + if (file === "bytes-after-deep-tail.msgpack") assert.ok(Buffer.from(bytes).includes(TAIL_CANARY), "the deep tail carries the canary the assertions look for"); + const payload = documentPayloadFromBytes(bytes); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(surface, payload), surface.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(plain, payload), plain.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(codecOnly, payload), codecOnly.type); + + // The store path: a single-file store holding the vector as a record payload refuses on load. + const path = join(dir, `${file}.cc`); + await new SingleFileMessagePackBackingStore(path).push(envelopeOf(bytes)); + const cache = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type); + + // The put path: an envelope arriving with the tail is refused and held nowhere. + const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([])).build(); + await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type); + } +}); + +test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => { + const whole = await read("union-surface.msgpack"); + const surface = surfaceDocument(); + for (const cut of [0, 1, 10, whole.length - 1]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(whole.slice(0, cut))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `truncated at ${cut}`, + ); + } + // A header cut short of its length is an early end too, whatever width it declares. + for (const early of ["dc00", "dd000000", "de00", "c4", "d9", "a5616263", "9291"]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(fromHex(early))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `early end ${early}`, + ); + } + const opaque = defineDocumentType({ type: "tests.opaque", schema: z.unknown(), formatter: { encode: (value: Uint8Array) => value, decode: (bytes: Uint8Array) => bytes } }); + const tailed = await read("bytes-after-nil.msgpack"); + assert.equal((decodeDocumentPayload(opaque, documentPayloadFromBytes(tailed)) as Uint8Array).length, tailed.length); +});