CultLib/packages/cultcache-ts/test/payload-depth.test.ts

import assert from "node:assert/strict";
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";

import { encode } from "@msgpack/msgpack";
import { z } from "zod";

import * as root from "../src";
import { inspectCultCacheBytes } from "../src/cult-cache-inspector";
import {
  CultCachePayloadDepthError,
  MAX_PAYLOAD_DEPTH,
  decodeDocumentPayload,
  defineDocumentType,
  documentPayloadBytes,
  documentPayloadFromBytes,
  encodeDocumentPayload,
} from "../src/document";
import * as codecs from "../src/codec";
import { checkPayloadDepth } from "../src/payload-depth";

const L = MAX_PAYLOAD_DEPTH;
const CANARY = "CANARY-payload-depth-91d3";

const cat = (...parts: Uint8Array[]) => Uint8Array.from(parts.flatMap((part) => [...part]));
const bytes = (...values: number[]) => Uint8Array.from(values);
const repeat = (unit: Uint8Array, times: number) => {
  const out = new Uint8Array(unit.length * times);
  for (let index = 0; index < times; index += 1) out.set(unit, index * unit.length);
  return out;
};
// n containers nested in one another around a leaf; `open` is one container's header with one child (two for a map).
const chain = (open: Uint8Array, n: number, leaf: Uint8Array = bytes(0xc0)) => cat(repeat(open, n), leaf);
const arrays = (n: number, leaf?: Uint8Array) => chain(bytes(0x91), n, leaf);

const refused = (payload: Uint8Array, label: string) =>
  assert.throws(() => checkPayloadDepth(payload), (error: unknown) =>
    error instanceof CultCachePayloadDepthError && error.depth === L + 1 && error.limit === L, label);

test("the limit is 256: depth 256 passes and 257 is refused, for every array and map header width", () => {
  assert.equal(L, 256);
  const kinds: [string, Uint8Array][] = [
    ["fixarray", bytes(0x91)],
    ["array16", bytes(0xdc, 0, 1)],
    ["array32", bytes(0xdd, 0, 0, 0, 1)],
    ["fixmap", bytes(0x81, 0x00)],
    ["map16", bytes(0xde, 0, 1, 0x00)],
    ["map32", bytes(0xdf, 0, 0, 0, 1, 0x00)],
  ];
  for (const [name, open] of kinds) {
    assert.doesNotThrow(() => checkPayloadDepth(chain(open, L)), `${name} at the limit`);
    refused(chain(open, L + 1), `${name} past the limit`);
  }
});

test("an empty map or array as the innermost container counts like any other", () => {
  for (const empty of [bytes(0x90), bytes(0x80), bytes(0xdc, 0, 0), bytes(0xdd, 0, 0, 0, 0), bytes(0xde, 0, 0), bytes(0xdf, 0, 0, 0, 0)]) {
    assert.doesNotThrow(() => checkPayloadDepth(arrays(L - 1, empty)));
    refused(arrays(L, empty), "empty innermost past the limit");
  }
  // [a chain of 255, a chain of 255]: the first sibling closes before the second opens, so the depth is 256, not 511.
  const sibling = arrays(L - 1);
  assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), sibling, sibling)));
  refused(cat(bytes(0x92), sibling, arrays(L)), "the second sibling is one deeper");
  // A map's key and value both count as elements: {key: chain, key: chain} closes after four elements.
  assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x82, 0x00), sibling, bytes(0x01), sibling)));
  refused(cat(bytes(0x82, 0x00), sibling, bytes(0x01), arrays(L)), "the second map value is one deeper");
  refused(cat(bytes(0x82), arrays(L), sibling, bytes(0x01), sibling), "a map key one deeper");
});

test("a header counts every child it announces, up to the widest of each kind", () => {
  const nils = (count: number) => repeat(bytes(0xc0), count);
  const kinds: [string, Uint8Array, number][] = [
    ["the longest fixarray", bytes(0x9f), 15],
    ["the longest fixmap", bytes(0x8f), 30],
    ["array16 of two", bytes(0xdc, 0, 2), 2],
    ["array32 of two", bytes(0xdd, 0, 0, 0, 2), 2],
    ["map16 of two pairs", bytes(0xde, 0, 2), 4],
    ["map32 of two pairs", bytes(0xdf, 0, 0, 0, 2), 4],
  ];
  for (const [name, header, elements] of kinds) {
    assert.doesNotThrow(() => checkPayloadDepth(cat(header, nils(elements - 1), arrays(L - 1))), `${name}: the last child at the limit`);
    refused(cat(header, nils(elements - 1), arrays(L)), `${name}: the last child past it`);
  }
});

test("every scalar is skipped by its length, whatever bytes it holds", () => {
  const filler = (count: number) => repeat(bytes(0x91), count);
  const scalars: [string, Uint8Array][] = [
    ["nil", bytes(0xc0)], ["the never-used 0xc1", bytes(0xc1)], ["false", bytes(0xc2)], ["true", bytes(0xc3)], ["positive fixint", bytes(0x7f)], ["negative fixint", bytes(0xe0)],
    ["uint8", bytes(0xcc, 0x91)], ["uint16", cat(bytes(0xcd), filler(2))], ["uint32", cat(bytes(0xce), filler(4))], ["uint64", cat(bytes(0xcf), filler(8))],
    ["int8", bytes(0xd0, 0x91)], ["int16", cat(bytes(0xd1), filler(2))], ["int32", cat(bytes(0xd2), filler(4))], ["int64", cat(bytes(0xd3), filler(8))],
    ["float32", cat(bytes(0xca), filler(4))], ["float64", cat(bytes(0xcb), filler(8))],
    ["fixext1", cat(bytes(0xd4), filler(2))], ["fixext2", cat(bytes(0xd5), filler(3))], ["fixext4", cat(bytes(0xd6), filler(5))],
    ["fixext8", cat(bytes(0xd7), filler(9))], ["fixext16", cat(bytes(0xd8), filler(17))],
    ["fixstr", cat(bytes(0xa3), filler(3))], ["the longest fixstr", cat(bytes(0xbf), filler(31))], ["str8", cat(bytes(0xd9, 3), filler(3))], ["str16", cat(bytes(0xda, 0, 3), filler(3))],
    ["str32", cat(bytes(0xdb, 0, 0, 0, 3), filler(3))],
    ["bin8", cat(bytes(0xc4, 3), filler(3))], ["bin16", cat(bytes(0xc5, 0, 3), filler(3))], ["bin32", cat(bytes(0xc6, 0, 0, 0, 3), filler(3))],
    ["ext8", cat(bytes(0xc7, 3, 0x7f), filler(3))], ["ext16", cat(bytes(0xc8, 0, 3, 0x7f), filler(3))],
    ["ext32", cat(bytes(0xc9, 0, 0, 0, 3, 0x7f), filler(3))],
    ["str16 of 300 bytes", cat(bytes(0xda, 0x01, 0x2c), filler(300))],
  ];
  for (const [name, scalar] of scalars) {
    assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), scalar, arrays(L - 1))), `${name} then a chain at the limit`);
    refused(cat(bytes(0x92), scalar, arrays(L)), `${name} then a chain past it`);
    // As the leaf of a chain: a scalar read at the wrong length leaves containers open, and the sibling chain then reads one deeper.
    assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), arrays(L - 1, scalar), arrays(L - 1))), `${name} closes the chain it ends`);
  }
});

test("a payload that ends early, is empty or carries trailing bytes is the reader's business, not the walk's", () => {
  for (const early of [[], [0xd9], [0xda, 0], [0xdb, 0, 0, 0], [0xc4], [0xc5, 0], [0xc6, 0, 0, 0], [0xc7], [0xc8, 0], [0xc9, 0, 0, 0], [0xdc, 0], [0xdd, 0, 0, 0], [0xde, 0], [0xdf, 0, 0, 0]]) {
    assert.doesNotThrow(() => checkPayloadDepth(bytes(...early)), early.map((value) => value.toString(16)).join(" "));
  }
  assert.doesNotThrow(() => checkPayloadDepth(arrays(L - 1, bytes(0x91))), "a chain cut off before its leaf");
  assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), "bytes after the first complete value are not walked");
  assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0)));
});

test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => {
  refused(arrays(100_000), "100000 nested arrays");
  refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps");
});

test("a refusal names the subject, the depth and the limit and never a byte of the payload", () => {
  const payload = chain(bytes(0x91), L + 1, cat(bytes(0xd9, CANARY.length), new TextEncoder().encode(CANARY)));
  assert.throws(
    () => checkPayloadDepth(payload, 'document type "t.note" payload'),
    (error: unknown) => error instanceof CultCachePayloadDepthError
      && error.name === "CultCachePayloadDepthError"
      && error.message === 'document type "t.note" payload nests 257 MessagePack containers deep; the limit is 256.',
  );
  const canaryPayload = payload;
  assert.throws(() => checkPayloadDepth(canaryPayload), (error: Error) => !error.message.includes(CANARY) && !JSON.stringify(error).includes(CANARY));
});

const note = z.object({ id: z.string(), count: z.number() });
const slotCalls: string[] = [];
const watched = { ...codecs.string, decode: (raw: unknown) => (slotCalls.push("decode"), codecs.string.decode(raw)), encode: (value: string) => (slotCalls.push("encode"), codecs.string.encode(value)) };
const guarded = defineDocumentType({
  type: "t.guarded",
  schemaId: "t.guarded.v0",
  schema: note,
  name: "id",
  codec: codecs.slots<{ id: string; count: number }>({ 0: ["id", watched], 1: ["count", codecs.int32] }, "T.Guarded"),
});

test("the owner refuses a payload past the limit before a codec reads it, and a codec's bytes before they are wrapped", () => {
  assert.throws(() => decodeDocumentPayload(guarded, documentPayloadFromBytes(arrays(L + 1))), CultCachePayloadDepthError);
  assert.deepEqual(slotCalls, [], "the codec never saw the deep payload");
  assert.deepEqual(decodeDocumentPayload(guarded, documentPayloadFromBytes(encode(["a", 1]))), { id: "a", count: 1 });

  const formatterBytes = (value: unknown) => documentPayloadBytes(encodeDocumentPayload(guarded, value));
  assert.deepEqual(formatterBytes({ id: "a", count: 1 }), encode(["a", 1]));
});

test("a document without a codec or formatter writes a value L deep and refuses L+1 by the walk", () => {
  const anything = defineDocumentType({ type: "t.any", schemaId: "t.any.v0", schema: z.any(), name: "id" });
  const nest = (n: number): unknown => (n === 1 ? [] : [nest(n - 1)]);
  const atLimit = encodeDocumentPayload(anything, nest(L));
  assert.deepEqual(documentPayloadBytes(atLimit), encode(nest(L), { maxDepth: 1000 }));
  assert.deepEqual(decodeDocumentPayload(anything, atLimit), nest(L));
  assert.throws(() => encodeDocumentPayload(anything, nest(L + 1)), (error: unknown) =>
    error instanceof CultCachePayloadDepthError && error.depth === L + 1 && error.limit === L
      && error.message === 'document type "t.any" value nests 257 MessagePack containers deep; the limit is 256.');
  assert.throws(() => decodeDocumentPayload(anything, documentPayloadFromBytes(arrays(L + 1))), CultCachePayloadDepthError);
});

test("the limit and its error are exported from the root, and the walk itself is not", () => {
  assert.equal(root.MAX_PAYLOAD_DEPTH, 256);
  assert.equal(root.CultCachePayloadDepthError, CultCachePayloadDepthError);
  assert.equal("checkPayloadDepth" in root, false);
});

// A v1 store whose one record carries `payload` as bin, under a catalog entry that names its schema.
async function storeWith(payload: Uint8Array): Promise<string> {
  const file = join(await mkdtemp(join(tmpdir(), "cultcache-payload-depth-")), "deep.cc");
  const entry = ["sha256:stale", "tests.schema_stamped_entry", "tests.schema_stamped_entry.v1", "stale", "", [], []];
  await writeFile(file, encode(["cultcache.store.v1", [entry], [["record-1", "sha256:stale", "2026-06-25T12:00:00Z", payload]]]));
  return file;
}
const stamped = (containersInsideTheStamp: number) =>
  cat(bytes(0x92), encode("tests.schema_stamped_entry.v1"), arrays(containersInsideTheStamp));

test("the inspector previews a payload at the limit and reports one past it, or 100000 deep, as a decode error", async () => {
  const atLimit = await storeWith(stamped(L - 1));
  const previewed = inspectCultCacheBytes(atLimit, await readFile(atLimit)).records[0]!;
  assert.equal(previewed.payloadDecodeError, undefined);
  assert.equal(previewed.schemaName, "tests.schema_stamped_entry");

  for (const inside of [L, 100_000]) {
    const file = await storeWith(stamped(inside));
    const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!;
    assert.equal(record.payloadDecodeError, "document payload nests 257 MessagePack containers deep; the limit is 256.");
    assert.equal(record.schemaName, "tests.schema_stamped_entry");
  }
});

// A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not.
const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff);

test("a 32-bit length or count is read whole: the walk skips a 65539-byte scalar and counts 65540 children", () => {
  const blob = repeat(bytes(0x91), 65539);
  const scalars: [string, Uint8Array][] = [
    ["bin32", cat(bytes(0xc6), be32(65539), blob)],
    ["str32", cat(bytes(0xdb), be32(65539), blob)],
    ["ext32", cat(bytes(0xc9), be32(65539), bytes(0x7f), blob)],
  ];
  for (const [name, scalar] of scalars) {
    assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), scalar, arrays(L - 1))), `${name} of 65539 bytes then a chain at the limit`);
    refused(cat(bytes(0x92), scalar, arrays(L)), `${name} of 65539 bytes then a chain past it`);
  }
  // Both counts exceed 65535 and their low sixteen bits are small (4 children; 1 pair), so a count cut to 16 bits closes the container early.
  const wide: [string, Uint8Array, number][] = [
    ["array32", cat(bytes(0xdd), be32(65540)), 65540],
    ["map32", cat(bytes(0xdf), be32(65537)), 2 * 65537],
  ];
  for (const [name, header, children] of wide) {
    const nils = repeat(bytes(0xc0), children);
    refused(cat(bytes(0x92), header, nils, arrays(L)), `${name} announcing more than 65535: the sibling chain past the limit`);
    assert.doesNotThrow(() => checkPayloadDepth(cat(bytes(0x92), header, nils, arrays(L - 1))), `${name}: the sibling chain at the limit`);
  }
});

test("a user formatter's bytes are its own: a 0x91 grey image round-trips, while plain and codec documents are still walked", () => {
  const grey = defineDocumentType({
    type: "t.grey",
    schemaId: "t.grey.v0",
    schema: z.object({ pixels: z.custom<Uint8Array>() }),
    name: "pixels",
    formatter: { encode: (value) => (value as { pixels: Uint8Array }).pixels, decode: (payload) => ({ pixels: payload }) },
  });
  for (const size of [300, 4096]) {
    const pixels = new Uint8Array(size).fill(0x91);
    const written = encodeDocumentPayload(grey, { pixels });
    assert.deepEqual(documentPayloadBytes(written), pixels, `${size} bytes written as the formatter made them`);
    assert.deepEqual(decodeDocumentPayload(grey, written), { pixels }, `${size} bytes read back through the formatter`);
  }

  const anything = defineDocumentType({ type: "t.any2", schemaId: "t.any2.v0", schema: z.any(), name: "id" });
  const nest = (n: number): unknown => (n === 1 ? [] : [nest(n - 1)]);
  assert.throws(() => encodeDocumentPayload(anything, nest(L + 1)), CultCachePayloadDepthError);
  assert.throws(() => decodeDocumentPayload(anything, documentPayloadFromBytes(arrays(L + 1))), CultCachePayloadDepthError);

  // A slot that writes `depth` nested arrays, so a codec document can be made as deep as the test needs.
  const nested: codecs.Codec<number> = {
    typeName: "System.Object",
    encode: (depth) => nest(depth),
    decode: (raw) => (raw as unknown[]).length,
  };
  const coded = defineDocumentType({
    type: "t.coded",
    schemaId: "t.coded.v0",
    schema: z.any(),
    name: "id",
    codec: codecs.slots<{ id: string; depth: number }>({ 0: ["id", codecs.string], 1: ["depth", nested] }, "T.Coded"),
  });
  assert.doesNotThrow(() => encodeDocumentPayload(coded, { id: "a", depth: L - 1 }));
  assert.throws(() => encodeDocumentPayload(coded, { id: "a", depth: L }), CultCachePayloadDepthError);
  assert.throws(() => decodeDocumentPayload(coded, documentPayloadFromBytes(arrays(L + 1))), CultCachePayloadDepthError);
});