From d1dfd8f0276d818cc30699eb985e1d6232c7b5ab Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Fri, 23 Oct 2026 01:38:16 +0200 Subject: [PATCH] cultcache-ts: the reader judges the stamp from the header before it decodes the body; the header-form pins return Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-ts/src/cult-cache-inspector.ts | 3 +- .../single-file-messagepack-backing-store.ts | 4 +- packages/cultcache-ts/src/store-format.ts | 40 +++++++++ packages/cultcache-ts/test/cult-cache.test.ts | 89 ++++++++++++++++++- 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index f3587317..1c359157 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -1,7 +1,7 @@ import { decode } from "@msgpack/msgpack"; import { checkPayloadDepth } from "./payload-depth"; -import { type StoreFormat, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format"; +import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; export interface CultCacheInspection { filePath: string; @@ -47,6 +47,7 @@ export function inspectCultCacheBytes( bytes: Uint8Array, fileSizeBytes = bytes.length, ): CultCacheInspection { + refuseUnreadStoreBytes(bytes); const decoded = decode(bytes); if (isStoreSnapshot(decoded)) { diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index d6e5d05c..2501c632 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -7,7 +7,7 @@ import { z } from "zod"; import { documentPayloadBytes, documentPayloadFromBytes } from "./document"; import { commitStoreFile, withStoreWriteLock } from "./store-file"; -import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format"; +import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, CultCacheEnvelope, @@ -400,6 +400,8 @@ export type SingleFileStoreRead = { }; function decodeStoreFile(data: Uint8Array): Omit { + // The stamp is judged before the body is decoded, as Rust's store_header does: a newer store is refused by name. + refuseUnreadStoreBytes(data); const decoded = decode(data); const snapshot = decodeSnapshot(decoded); if (snapshot) { diff --git a/packages/cultcache-ts/src/store-format.ts b/packages/cultcache-ts/src/store-format.ts index 910d1a90..193abc03 100644 --- a/packages/cultcache-ts/src/store-format.ts +++ b/packages/cultcache-ts/src/store-format.ts @@ -1,3 +1,5 @@ +import { decode } from "@msgpack/msgpack"; + export const STORE_FORMAT_VERSION = "cultcache.store.v1"; /** @@ -26,6 +28,44 @@ export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...u return true; } +/** + * Refuses a store whose header names a format this runtime does not read, judging the header alone: the leading array's first + * element, when it is a complete string. The reader calls it before it decodes the body, so a newer store is refused by name + * whether or not its body is whole. Bytes with no such header (a legacy array, an empty, truncated or garbled file) are not + * refused here. + */ +export function refuseUnreadStoreBytes(bytes: Uint8Array): void { + const header = peekHeader(bytes); + if (header !== undefined && header.startsWith(STORE_FORMAT_PREFIX)) { + refuseUnreadFormat(header); + } +} + +// The first element of a leading non-empty MessagePack array when it is a complete string; anything else is undefined. The +// string is decoded by the decoder a read uses, so a stamp gets one verdict from the peek and from the whole decode. +function peekHeader(bytes: Uint8Array): string | undefined { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const lead = bytes[0]; + let offset: number; + if (lead === undefined) return undefined; + if (lead >= 0x91 && lead <= 0x9f) offset = 1; + else if (lead === 0xdc && bytes.length >= 3 && view.getUint16(1) > 0) offset = 3; + else if (lead === 0xdd && bytes.length >= 5 && view.getUint32(1) > 0) offset = 5; + else return undefined; + const start = offset; + const tag = bytes[offset]; + let length: number; + if (tag === undefined) return undefined; + if (tag >= 0xa0 && tag <= 0xbf) { length = tag & 0x1f; offset += 1; } + else if (tag === 0xd9 && bytes.length >= offset + 2) { length = bytes[offset + 1]!; offset += 2; } + else if (tag === 0xda && bytes.length >= offset + 3) { length = view.getUint16(offset + 1); offset += 3; } + else if (tag === 0xdb && bytes.length >= offset + 5) { length = view.getUint32(offset + 1); offset += 5; } + else return undefined; + if (bytes.length < offset + length) return undefined; + const header = decode(bytes.subarray(start, offset + length)); + return typeof header === "string" ? header : undefined; +} + /** The one verdict on a stamp: this runtime reads v1 and v3, and any other `cultcache.store.*` header is refused by name. */ function refuseUnreadFormat(header: string): void { if (header !== STORE_FORMAT_VERSION && header !== STORE_FORMAT_ELEMENT_IDS) { diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 9d804400..6f2796e0 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -15,7 +15,7 @@ import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; import { SingleFileMessagePackBackingStore, readStoreBytes } from "../src/single-file-messagepack-backing-store"; -import { SchemaConflictError, StoreUnreadableError } from "../src/store-format"; +import { SchemaConflictError, StoreUnreadableError, refuseUnreadStoreBytes } from "../src/store-format"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; const execFileAsync = promisify(execFile); @@ -2083,6 +2083,93 @@ test("SingleFileMessagePackBackingStore refusal says what was found and names ev } }); +function headerRefused(bytes: Uint8Array): boolean { + try { + refuseUnreadStoreBytes(bytes); + return false; + } catch (error) { + assert.match((error as Error).message, /not one this runtime reads/u); + return true; + } +} + +function storeStart(arrayLead: number[], text: string, strForm: "fix" | "8" | "16" | "32"): Uint8Array { + const body = Buffer.from(text, "utf8"); + const n = body.length; + const tag = + strForm === "fix" ? [0xa0 | n] + : strForm === "8" ? [0xd9, n] + : strForm === "16" ? [0xda, n >> 8, n & 0xff] + : [0xdb, 0, 0, n >> 8, n & 0xff]; + return new Uint8Array([...arrayLead, ...tag, ...body]); +} + +test("a store's header is judged from its first bytes, in every MessagePack array and string form", () => { + const leads = [[0x91], [0x9f], [0xdc, 0x00, 0x01], [0xdc, 0x01, 0x00], [0xdd, 0x00, 0x00, 0x00, 0x01], [0xdd, 0x00, 0x01, 0x00, 0x00]]; + for (const lead of leads) { + for (const form of ["fix", "8", "16", "32"] as const) { + const v4 = storeStart(lead, "cultcache.store.v4", form); + assert.equal(headerRefused(v4), true, `${lead} ${form} v4`); + assert.equal(headerRefused(v4.subarray(0, v4.length - 1)), false, `${lead} ${form} v4 one byte short`); + assert.equal(headerRefused(storeStart(lead, "cultcache.store.v1", form)), false, `${lead} ${form} v1`); + assert.equal(headerRefused(storeStart(lead, "cultcache.store.v3", form)), false, `${lead} ${form} v3`); + assert.equal(headerRefused(storeStart(lead, "other.store.v4", form)), false, `${lead} ${form} other`); + assert.equal(headerRefused(storeStart(lead, "", form)), false, `${lead} ${form} empty`); + } + } + // The longest fixstr, and a string long enough to need str16. + assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v4${"x".repeat(13)}`, "fix")), true); + assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v4${"x".repeat(300)}`, "16")), true); + assert.equal(headerRefused(storeStart([0x91], `cultcache.store.v4${"x".repeat(300)}`, "32")), true); + // A view into a larger buffer is read from its own start. + const padded = new Uint8Array([9, 9, ...storeStart([0x91], "cultcache.store.v4", "fix")]); + assert.equal(headerRefused(padded.subarray(2)), true); +}); + +test("bytes with no readable store stamp are not refused", () => { + const stamp = [...Buffer.from("cultcache.store.v4", "utf8")]; + const cases: Record = { + empty: [], + "empty array": [0x90], + "empty array16": [0xdc, 0x00, 0x00, 0xb2, ...stamp], + "empty array32": [0xdd, 0x00, 0x00, 0x00, 0x00, 0xb2, ...stamp], + "array16 cut before its count": [0xdc, 0x00], + "array32 cut before its count": [0xdd, 0x00, 0x00, 0x00], + "array with no element": [0x91], + "first element is not a string": [0x92, 0x01, 0xb2, ...stamp], + "a bare string": [0xb2, ...stamp], + "a map": [0x81, 0xb2, ...stamp], + "str8 cut before its length": [0x91, 0xd9], + "str16 cut before its length": [0x91, 0xda, 0x00], + "str32 cut before its length": [0x91, 0xdb, 0x00, 0x00, 0x00], + }; + for (const [name, bytes] of Object.entries(cases)) { + assert.equal(headerRefused(new Uint8Array(bytes)), false, name); + } +}); + +test("a newer store is refused by name whatever its body: truncated, undecodable or trailing, by a read, a whole write and the inspector", async () => { + const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; + const whole = encode(["cultcache.store.v4", [], [["k", "s", "t", new Uint8Array([0x90])]]]); + const bodies: Record = { + truncated: whole.subarray(0, whole.length - 3), + "header only": whole.subarray(0, 20), + undecodable: Uint8Array.of(...whole.subarray(0, 20), 0xc1), + trailing: Uint8Array.of(...whole, 0xff), + whole, + }; + for (const [name, bytes] of Object.entries(bodies)) { + const file = join(await mkdtemp(join(tmpdir(), "cultcache-newer-")), "store.msgpack"); + await writeFile(file, bytes); + const store = new SingleFileMessagePackBackingStore(file); + await assert.rejects(() => store.pullAll(), /not one this runtime reads/u, `${name} read`); + await assert.rejects(() => store.pushAll([envelope]), /not one this runtime reads/u, `${name} pushAll`); + await assert.rejects(() => store.pushAll([envelope], { soft: true }), /not one this runtime reads/u, `${name} soft pushAll`); + assert.throws(() => inspectCultCacheBytes(file, bytes), /not one this runtime reads/u, `${name} inspect`); + assert.deepEqual([...(await readFile(file))], [...bytes], `${name} left as it was`); + } +}); + test("a stamp this runtime reads, followed by bytes that are not UTF-8, gets the same refusal from a read and from a whole write", async () => { const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; for (const header of ["cultcache.store.v1", "cultcache.store.v3"]) {