From be968a71e614202f538d4efabec0db20df983018 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:08:24 +0000 Subject: [PATCH 1/5] Bytes after the value: a payload is one MessagePack value (TypeScript) decodeDocumentPayload refuses bytes after the first value by count, from the checkPayloadDepth walk's end, with the shared clause; readWire's own trailing check is deleted. Walks bytes-after-outcomes.msgpack on the plain, codec, store and putEnvelope paths. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 2 + packages/cultcache-ts/README.md | 6 +- packages/cultcache-ts/src/document.ts | 8 +- packages/cultcache-ts/src/payload-depth.ts | 28 +++--- packages/cultcache-ts/src/wire.ts | 5 +- .../cultcache-ts/test/payload-depth.test.ts | 10 +++ .../cultcache-ts/test/union-parity.test.ts | 89 ++++++++++++++++++- 7 files changed, 128 insertions(+), 20 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index e71821c8..e45dc14e 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,9 +20,11 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. +- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication), refuses bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse, and a user formatter's bytes are not walked. The refused set is unchanged, but the message was `@msgpack`'s `RangeError` ("Extra N of M byte(s)") for a plain document and the codec reader's "bytes after its value" for a codec document, so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Implied bump: 0.16.0 (pre-1.0 minor). ### Added +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (it was `void`); it still refuses only depth. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 2096d258..041e90ae 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -49,7 +49,9 @@ through `decodeDocumentPayload`, and when you need the bytes themselves (hashing write) take them with `documentPayloadBytes`, the sanctioned escape. - `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter, - codec or plain map and then its schema. The definition is required. + codec or plain map and then its schema. The definition is required. A payload is exactly one + MessagePack value: bytes after the first are refused by count, naming the type, and never walked or + echoed (a payload that ends early is the decoder's to refuse). - `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter, codec or plain map. - `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a @@ -327,7 +329,7 @@ If you do not supply one, `CultCacheTS` uses MessagePack plus Zod validation by The 256-container nesting bound (`MAX_PAYLOAD_DEPTH`, refused with `CultCachePayloadDepthError`) covers the MessagePack the library itself writes: the default format and a codec. A payload your formatter writes is opaque bytes in any format, so the -library does not walk it as MessagePack; its format and its bounds are your formatter's. +library does not walk it as MessagePack; its format, its bounds and whether it is one value are your formatter's. Zod is no longer mandatory. Any schema object with a `parse(...)` function will do, which means generated Ajv-backed JSON-schema contracts can feed the cache diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 10af96a6..3443451e 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -119,7 +119,13 @@ export function decodeDocumentPayload { const bytes = documentPayloadBytes(payload); - if (writesLibraryMessagePack(definition)) checkPayloadDepth(bytes, `document type "${definition.type}" payload`); + if (writesLibraryMessagePack(definition)) { + // A payload is exactly one MessagePack value: bytes after the first are refused by count, from the walk's end, before + // any formatter reads; the tail is never walked or echoed. A payload that ends early stays the decoder's to refuse. + const subject = `document type "${definition.type}" payload`; + const end = checkPayloadDepth(bytes, subject); + if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + } const decoded = resolveDocumentFormatter(definition).decode(bytes); return definition.schema.parse(decoded) as CultCacheDocumentValue; } diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts index 40ca7afe..efbc3a74 100644 --- a/packages/cultcache-ts/src/payload-depth.ts +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -12,8 +12,9 @@ export class CultCachePayloadDepthError extends Error { // Walks the headers of one MessagePack value iteratively (a count stack bounded by the limit, scalars skipped by length, // nothing read) and throws at the first container past the limit. The message names the subject, the depth and the limit, -// never payload bytes. A payload that ends early is the reader's to refuse. -export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): void { +// never payload bytes. Returns the offset where the first value ends, or undefined when the payload ends early (the +// reader's to refuse); what follows that offset is never walked, and whether it may exist is the caller's rule. +export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): number | undefined { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const remaining = new Float64Array(MAX_PAYLOAD_DEPTH); let depth = 0; @@ -33,12 +34,12 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code <= 0x8f) { container = true; header = 1; children = 2 * (code & 0x0f); } else if (code <= 0x9f) { container = true; header = 1; children = code & 0x0f; } else if (code <= 0xbf) skip = 1 + (code & 0x1f); - else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return; skip = 2 + n; } - else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return; skip = 5 + n; } - else if (code === 0xc7) { const n = width(1); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc8) { const n = width(2); if (n === undefined) return; skip = 4 + n; } - else if (code === 0xc9) { const n = width(4); if (n === undefined) return; skip = 6 + n; } + else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return undefined; skip = 2 + n; } + else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return undefined; skip = 5 + n; } + else if (code === 0xc7) { const n = width(1); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc8) { const n = width(2); if (n === undefined) return undefined; skip = 4 + n; } + else if (code === 0xc9) { const n = width(4); if (n === undefined) return undefined; skip = 6 + n; } else if (code === 0xca || code === 0xce || code === 0xd2) skip = 5; else if (code === 0xcb || code === 0xcf || code === 0xd3) skip = 9; else if (code === 0xcc || code === 0xd0) skip = 2; @@ -47,10 +48,10 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code === 0xd6) skip = 6; else if (code === 0xd7) skip = 10; else if (code === 0xd8) skip = 18; - else if (code === 0xdc) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = n; } - else if (code === 0xdd) { const n = width(4); if (n === undefined) return; container = true; header = 5; children = n; } - else if (code === 0xde) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = 2 * n; } - else { const n = width(4); if (n === undefined) return; container = true; header = 5; children = 2 * n; } + else if (code === 0xdc) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = n; } + else if (code === 0xdd) { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = n; } + else if (code === 0xde) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = 2 * n; } + else { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = 2 * n; } if (container) { if (depth === MAX_PAYLOAD_DEPTH) throw new CultCachePayloadDepthError(MAX_PAYLOAD_DEPTH + 1, MAX_PAYLOAD_DEPTH, subject); @@ -65,6 +66,7 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload // One element is complete; close every container it was the last element of. while (depth > 0 && --remaining[depth - 1] === 0) depth--; - if (depth === 0) return; + if (depth === 0) return position; } + return undefined; } diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index 12bf572a..d351c446 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -128,7 +128,6 @@ export function readWire(payload: Uint8Array): unknown { } } - const tree = next(); - if (offset !== payload.length) throw new Error("the MessagePack payload has bytes after its value."); - return tree; + // One value is read; the caller owns what follows it (decodeDocumentPayload refuses a payload with a tail). + return next(); } diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index 382f0672..e792accb 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -119,6 +119,16 @@ test("a payload that ends early, is empty or carries trailing bytes is the reade assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0))); }); +test("the walk returns where the first value ends, and undefined when the payload ends early", () => { + assert.equal(checkPayloadDepth(bytes(0xc0)), 1); + assert.equal(checkPayloadDepth(bytes(0xc0, 0xc1, 0xc1)), 1, "a top-level scalar ends at its own length; the tail is not judged"); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01, 0xa1, 0x61, 0xc0)), 5 - 1, "a container ends when its last child completes"); + assert.equal(checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), 1, "the tail is never walked"); + assert.equal(checkPayloadDepth(bytes()), undefined); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined); + assert.equal(checkPayloadDepth(bytes(0xd9)), undefined); +}); + test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { refused(arrays(100_000), "100000 nested arrays"); refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps"); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 46dcb42f..f71924cc 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -515,7 +515,7 @@ test("the wire reader keeps 64-bit integers whole, tells a float from an integer assert.throws(() => readWire(fromHex("dd")), /truncated/u); assert.throws(() => readWire(fromHex("ddffffffff")), /truncated/u, "a huge array header reserves nothing"); assert.throws(() => readWire(fromHex("a5616263")), /truncated/u); - assert.throws(() => readWire(fromHex("0102")), /bytes after its value/u); + assert.equal(readWire(fromHex("0102")), 1, "the reader reads one value; the document owner refuses what follows"); assert.throws(() => readWire(fromHex("81a161a162")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("c1")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("d4010a")), /map, extension or reserved/u); @@ -915,3 +915,90 @@ test("a cache load refuses a store whose record is past the limit, naming the ty const refusing = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(await read("depth-past-limit.msgpack"))])).build(); await assert.rejects(() => refusing.pullAllBackingStores(), (error: Error) => error instanceof CultCachePayloadDepthError && !error.message.includes(CANARY)); }); + +// ---- bytes-after vectors: a payload is exactly one MessagePack value (tests/vectors/union-parity/README.md) ---- + +// The C# reference writes the vectors: union-surface.msgpack followed by a tail the first value does not own. The one +// owner refuses by count, naming the type, from the walk's end; it never walks or echoes the tail. +async function bytesAfterEntries() { + const raw = decode(await read("bytes-after-outcomes.msgpack")) as unknown[][]; + return raw.map(([file, outcome, unionName, key, detail]) => ({ file: file as string, outcome: outcome as string, unionName: unionName as string, key: key as number | null, detail: detail as string })); +} + +// The C# vector writer's canary (BytesAfterVectorTests): it sits in the deep tail, so an echo of the tail would carry it. +const TAIL_CANARY = "CANARY-bytes-after-7d1e"; +const valueLength = async () => (await read("union-surface.msgpack")).length; +const plainDocument = () => defineDocumentType({ type: "tests.plain_surface", schema: z.unknown() }); +const codecDocument = () => defineDocumentType({ type: "tests.codec_surface", schema: surfaceSchema, codec: surfaceCodec }); + +async function assertBytesAfterRefused(file: string, tail: number, run: () => unknown | Promise, type: string) { + const clause = `${type}`; + await assert.rejects( + async () => run(), + (error: Error) => { + assert.ok(!(error instanceof CultCachePayloadDepthError), `${file}: the tail was walked`); + assert.ok(!error.message.includes(CANARY) && !error.message.includes(TAIL_CANARY), `${file}: the refusal echoed the tail`); + assert.ok(!error.message.includes("the limit is"), `${file}: the depth refusal`); + assert.ok(error.message.includes(clause), `${file}: names the type: ${error.message}`); + assert.ok(error.message.includes(`${tail} bytes after its first MessagePack value`), `${file}: names the count: ${error.message}`); + return true; + }, + file, + ); +} + +test("every bytes-after vector is refused by the one owner, by count, on the codec, plain and store paths", async () => { + const entries = await bytesAfterEntries(); + assert.equal(entries.length, 4); + const length = await valueLength(); + const surface = surfaceDocument(); + const plain = plainDocument(); + const codecOnly = codecDocument(); + const dir = await mkdtemp(join(tmpdir(), "bytes-after-")); + for (const { file, outcome, unionName, detail } of entries) { + assert.equal(outcome, "refused"); + assert.equal(unionName, "UnionSurface"); + assert.equal(detail, "bytes-after"); + const bytes = await read(file); + const tail = bytes.length - length; + assert.ok(tail > 0, file); + if (file === "bytes-after-deep-tail.msgpack") assert.ok(Buffer.from(bytes).includes(TAIL_CANARY), "the deep tail carries the canary the assertions look for"); + const payload = documentPayloadFromBytes(bytes); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(surface, payload), surface.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(plain, payload), plain.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(codecOnly, payload), codecOnly.type); + + // The store path: a single-file store holding the vector as a record payload refuses on load. + const path = join(dir, `${file}.cc`); + await new SingleFileMessagePackBackingStore(path).push(envelopeOf(bytes)); + const cache = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type); + + // The put path: an envelope arriving with the tail is refused and held nowhere. + const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([])).build(); + await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type); + } +}); + +test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => { + const whole = await read("union-surface.msgpack"); + const surface = surfaceDocument(); + for (const cut of [0, 1, 10, whole.length - 1]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(whole.slice(0, cut))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `truncated at ${cut}`, + ); + } + // A header cut short of its length is an early end too, whatever width it declares. + for (const early of ["dc00", "dd000000", "de00", "c4", "d9", "a5616263", "9291"]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(fromHex(early))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `early end ${early}`, + ); + } + const opaque = defineDocumentType({ type: "tests.opaque", schema: z.unknown(), formatter: { encode: (value: Uint8Array) => value, decode: (bytes: Uint8Array) => bytes } }); + const tailed = await read("bytes-after-nil.msgpack"); + assert.equal((decodeDocumentPayload(opaque, documentPayloadFromBytes(tailed)) as Uint8Array).length, tailed.length); +}); From 50a8ead871266672404073961d3eab7f0d6294da Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:34:35 +0000 Subject: [PATCH 2/5] Walk end is undefined for any early end; the inspector speaks the one clause; changelog files the entry under Changed Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 4 ++-- .../cultcache-ts/src/cult-cache-inspector.ts | 18 ++++++++++++++---- packages/cultcache-ts/src/document.ts | 9 +++------ packages/cultcache-ts/src/payload-depth.ts | 11 +++++++++++ .../cultcache-ts/test/payload-depth.test.ts | 10 ++++++++++ 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index e45dc14e..c43d17bc 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,15 +20,15 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. -- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication), refuses bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse, and a user formatter's bytes are not walked. The refused set is unchanged, but the message was `@msgpack`'s `RangeError` ("Extra N of M byte(s)") for a plain document and the codec reader's "bytes after its value" for a codec document, so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Implied bump: 0.16.0 (pre-1.0 minor). ### Added -- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (it was `void`); it still refuses only depth. +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. `checkPayloadIsOneValue(bytes, subject)` is the depth walk plus the bytes-after refusal. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed +- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication, the provider command), and the inspector refuse bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse. The refused set is unchanged; only the text changed (it was `@msgpack`'s `RangeError` "Extra N of M byte(s)" for a plain document and the codec reader's "bytes after its value" for a codec document, and the inspector's preview no longer shows a tail), so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Stated limit: a definition with its own `formatter` is not walked, so whether its payload is one value is the formatter's. Implied bump: 0.16.0 (pre-1.0 minor, with the Breaking entries above). - Opening a legacy (pre-v1) store is a pure read: payloads that were not bytes are normalized in memory and the file is left as it was, so a codec that refuses the content cannot lose the old bytes. The next push, delete or flush rewrites the file as v1, merging what the open read. diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index 1c359157..71643913 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -1,6 +1,6 @@ import { decode } from "@msgpack/msgpack"; -import { checkPayloadDepth } from "./payload-depth"; +import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth"; import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; export interface CultCacheInspection { @@ -235,12 +235,22 @@ function decodeLegacyEnvelope(value: unknown): InspectedRecord { }; } -// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it. +// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it, +// and bytes after the first value are refused with the shared clause (the tail is never decoded or shown). function decodeBoundedPayload(payload: Uint8Array): unknown { - checkPayloadDepth(payload); + checkPayloadIsOneValue(payload, "document payload"); return decode(payload); } +// Where the first value ends (the whole payload when it ends early or cannot be walked): a preview never shows a tail. +function valueEnd(payload: Uint8Array): number { + try { + return checkPayloadDepth(payload) ?? payload.length; + } catch { + return payload.length; + } +} + function previewPayload(payload: Uint8Array): Pick { try { return { payloadPreview: toJsonSafe(decodeBoundedPayload(payload)) }; @@ -248,7 +258,7 @@ function previewPayload(payload: Uint8Array): Pick value.toString(16).padStart(2, "0")) .join(" "), }, diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 3443451e..ac6e5f7d 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -1,5 +1,5 @@ import { resolveDocumentFormatter } from "./document-formatter"; -import { checkPayloadDepth } from "./payload-depth"; +import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth"; import type { AnyCultCacheDocumentDefinition, CultCacheSchemaCatalogMember, @@ -120,11 +120,8 @@ export function decodeDocumentPayload { const bytes = documentPayloadBytes(payload); if (writesLibraryMessagePack(definition)) { - // A payload is exactly one MessagePack value: bytes after the first are refused by count, from the walk's end, before - // any formatter reads; the tail is never walked or echoed. A payload that ends early stays the decoder's to refuse. - const subject = `document type "${definition.type}" payload`; - const end = checkPayloadDepth(bytes, subject); - if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + // A payload is exactly one MessagePack value: bytes after the first are refused by count before any formatter reads. + checkPayloadIsOneValue(bytes, `document type "${definition.type}" payload`); } const decoded = resolveDocumentFormatter(definition).decode(bytes); return definition.schema.parse(decoded) as CultCacheDocumentValue; diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts index efbc3a74..45b8d5ca 100644 --- a/packages/cultcache-ts/src/payload-depth.ts +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -64,9 +64,20 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload position += skip; } + // A scalar whose declared length runs past the payload is an early end, not a value. + if (position > bytes.length) return undefined; // One element is complete; close every container it was the last element of. while (depth > 0 && --remaining[depth - 1] === 0) depth--; if (depth === 0) return position; } return undefined; } + +// The one rule that a payload is exactly one MessagePack value: the depth walk, then a refusal of bytes after the first +// value, by count, naming the subject, from the walk's end. The tail is never walked or echoed; a payload that ends early +// stays the decoder's to refuse. decodeDocumentPayload and the inspector speak this clause and no other. +export function checkPayloadIsOneValue(bytes: Uint8Array, subject: string): number | undefined { + const end = checkPayloadDepth(bytes, subject); + if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + return end; +} diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index e792accb..6fb81265 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -127,6 +127,9 @@ test("the walk returns where the first value ends, and undefined when the payloa assert.equal(checkPayloadDepth(bytes()), undefined); assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined); assert.equal(checkPayloadDepth(bytes(0xd9)), undefined); + for (const cut of [[0xcc], [0xcb, 0x3f, 0xf8], [0xa5, 0x61, 0x62, 0x63], [0xc4, 0x02, 0x01], [0xd7, 0x01]]) { + assert.equal(checkPayloadDepth(bytes(...cut)), undefined, `a scalar cut short: ${cut.map((value) => value.toString(16)).join(" ")}`); + } }); test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { @@ -208,6 +211,13 @@ test("the inspector previews a payload at the limit and reports one past it, or } }); +test("the inspector refuses bytes after a payload's first value with the shared clause and previews no tail", async () => { + const file = await storeWith(bytes(0xc0, 0xc1, 0xc1)); + const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!; + assert.equal(record.payloadDecodeError, "document payload has 2 bytes after its first MessagePack value."); + assert.deepEqual(record.payloadPreview, { bytes: 3, hexPrefix: "c0" }); +}); + // A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not. const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff); From e6bd81adcb03cc80934e67c05d3b9998f0089d0c Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:34:35 +0000 Subject: [PATCH 3/5] Pin the tail refusal on CultNet replication, the provider command and the CDN manifest; held-nowhere is asserted Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-ts/test/union-parity.test.ts | 5 +++- .../test/provider-session-wire.test.ts | 13 +++++++++++ packages/cultnet-ts/test/content.test.ts | 7 ++++++ packages/cultnet-ts/test/cultnet.test.ts | 23 +++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index f71924cc..00554bc3 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -975,8 +975,11 @@ test("every bytes-after vector is refused by the one owner, by count, on the cod await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type); // The put path: an envelope arriving with the tail is refused and held nowhere. - const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([])).build(); + const written: CultCacheEnvelope[] = []; + const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build(); await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type); + assert.equal(written.length, 0, `${file}: nothing reached the store`); + assert.equal(held.get(surface, "surface"), undefined, `${file}: nothing is held in memory`); } }); diff --git a/packages/cultmesh-ts/test/provider-session-wire.test.ts b/packages/cultmesh-ts/test/provider-session-wire.test.ts index 4905f161..69314fbe 100644 --- a/packages/cultmesh-ts/test/provider-session-wire.test.ts +++ b/packages/cultmesh-ts/test/provider-session-wire.test.ts @@ -83,3 +83,16 @@ test("a provider command that is not a map, or lacks a required field, is refuse (error: Error) => error.message === "CultMesh provider command must be a MessagePack map.", ); }); + +test("a provider command payload with bytes after its first value is refused by count, without echoing the tail", () => { + const command = { commandId: "c1", commandKind: "k", providerId: "p", serviceInstanceId: "s", payload: { n: 1 } }; + const whole = documentPayloadBytes(encodeDocumentPayload(providerCommandDocument, command)); + const tailed = documentPayloadFromBytes(Uint8Array.from([...whole, ...Buffer.from("canary-tail-8c1f")])); + assert.throws( + () => decodeProviderCommandDocument({ + schemaId: cultMeshProviderSessionSchemas.command, recordKey: "p/s/c1", storedAt: "2026-10-11T00:00:00Z", + payloadEncoding: "messagepack" as const, payload: tailed, + }), + (error: Error) => error.message.includes("16 bytes after its first MessagePack value") && !error.message.includes("canary-tail-8c1f"), + ); +}); diff --git a/packages/cultnet-ts/test/content.test.ts b/packages/cultnet-ts/test/content.test.ts index ec23b64e..e2ae5ed9 100644 --- a/packages/cultnet-ts/test/content.test.ts +++ b/packages/cultnet-ts/test/content.test.ts @@ -392,6 +392,13 @@ test("decodeManifest accepts and refuses exactly what the reference does, and re } }); +test("a manifest with bytes after its first value is refused by count, whatever the tail is", () => { + const whole = encodeManifest(manifestOf()); + for (const tail of [[0xc0], [0xc1, 0xc1], [0x90]]) { + assert.throws(() => decodeManifest(Uint8Array.from([...whole, ...tail])), new RegExp(`${tail.length} bytes after its first MessagePack value`)); + } +}); + test("a manifest written as a map names the manifest, not a field inside it", () => { assert.throws(() => decodeManifest(encode({ 0: "a", 9: { k: "v" } })), /field 'manifest' has the wrong type/); }); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index ac4f539d..581fd66a 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -14,6 +14,8 @@ import { SchemaConflictError, SingleFileMessagePackBackingStore, defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, schemaIdentityOf, } from "@gamecult/cultcache-ts"; @@ -3140,3 +3142,24 @@ test("CultNet registry names the canonical schema id of a compatible alias, and assert.equal(registry.canonicalSchemaId("p3.nobody"), undefined); assert.equal(registry.canonicalSchemaId("p3.shared"), undefined, "an id several bindings claim resolves to none"); }); + +test("a replicated record with bytes after its first MessagePack value is refused by count and held nowhere", async () => { + const definition = defineDocumentType({ type: "tails.note", schemaId: "tails.note.v1", schema: z.object({ body: z.string() }) }); + const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition })]); + const origin = CultCache.builder().withDocumentType(definition).build(); + await origin.put(definition, "k", { body: "x" }); + const document = registry.createRawSnapshotResponse(origin, "tails").documents[0]!; + const whole = documentPayloadBytes(document.payload); + const target = CultCache.builder().withDocumentType(definition).build(); + await assert.rejects( + registry.applyRawDocumentPutMessage(target, { + schemaVersion: "cultnet.document_put_raw.v0", + messageId: "tails", + document: { ...document, payload: documentPayloadFromBytes(Uint8Array.from([...whole, 0xc0, 0xc1])) }, + }), + (error: Error) => error.message.includes('document type "tails.note" payload has 2 bytes after its first MessagePack value'), + ); + assert.deepEqual(target.snapshot(), []); + await registry.applyRawDocumentPutMessage(target, { schemaVersion: "cultnet.document_put_raw.v0", messageId: "whole", document }); + assert.deepEqual(target.getRequired(definition, "k"), { body: "x" }); +}); From 8c36ce4771b7467dc9789949fcc06a00a7b3a052 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:41:47 +0000 Subject: [PATCH 4/5] Changelog: the one-value check is internal, not new API Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index c43d17bc..9f0301cc 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -23,7 +23,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Added -- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. `checkPayloadIsOneValue(bytes, subject)` is the depth walk plus the bytes-after refusal. +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. The bytes-after refusal is internal (`checkPayloadIsOneValue`, not exported); no new public name. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed From 34d85b2b597e316265faa208a3ffbad6178ae4b2 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 05:06:33 +0000 Subject: [PATCH 5/5] Pin: a tail on a payload with unread slots is refused before any slot is marked or the record held Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-ts/test/union-parity.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 7c6534b4..d917ea7a 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -793,6 +793,27 @@ test("every bytes-after vector is refused by the one owner, by count, on the cod } }); +test("a payload with an unread slot and a tail is refused by count before any slot is marked or the record held", async () => { + const withSlot = await read("extra-slot-record.msgpack"); + const surface = surfaceDocument(); + const tailed = Uint8Array.from([...withSlot, 0xc0]); + const whole = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(withSlot)])).build(); + await whole.pullAllBackingStores(); + assert.ok(whole.unreadSlots(surface, "surface"), "without a tail the record is held with its unread slots"); + + const loading = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(tailed)])).build(); + await assertBytesAfterRefused("extra-slot-record+tail", 1, () => loading.pullAllBackingStores(), surface.type); + assert.equal(loading.get(surface, "surface"), undefined); + assert.equal(loading.unreadSlots(surface, "surface"), undefined); + + const written: CultCacheEnvelope[] = []; + const putting = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build(); + await assertBytesAfterRefused("extra-slot-record+tail", 1, () => putting.putEnvelope(surface, envelopeOf(tailed)), surface.type); + assert.equal(written.length, 0); + assert.equal(putting.get(surface, "surface"), undefined); + assert.equal(putting.unreadSlots(surface, "surface"), undefined); +}); + test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => { const whole = await read("union-surface.msgpack"); const surface = surfaceDocument();