From 2e9a44925cd2e52986aa368a2c80e634de01bcd8 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Mon, 19 Oct 2026 15:28:02 +0200 Subject: [PATCH] cultmesh-ts/content: a chunk handler that verifies before serving, a read-only directory store, a node:http listener Builds. The ./content subpath is the only way to reach it, so the package root gains no node:fs import. Co-Authored-By: Claude Opus 5.5 --- packages/cultmesh-ts/package.json | 5 + packages/cultmesh-ts/src/content.ts | 109 ++++++++++ packages/cultmesh-ts/test/content.test.ts | 233 ++++++++++++++++++++++ 3 files changed, 347 insertions(+) create mode 100644 packages/cultmesh-ts/src/content.ts create mode 100644 packages/cultmesh-ts/test/content.test.ts diff --git a/packages/cultmesh-ts/package.json b/packages/cultmesh-ts/package.json index b53555be..4d454b24 100644 --- a/packages/cultmesh-ts/package.json +++ b/packages/cultmesh-ts/package.json @@ -9,6 +9,11 @@ "types": "./dist/index.d.ts", "require": "./dist/index.js", "default": "./dist/index.js" + }, + "./content": { + "types": "./dist/content.d.ts", + "require": "./dist/content.js", + "default": "./dist/content.js" } }, "files": [ diff --git a/packages/cultmesh-ts/src/content.ts b/packages/cultmesh-ts/src/content.ts new file mode 100644 index 00000000..500075fd --- /dev/null +++ b/packages/cultmesh-ts/src/content.ts @@ -0,0 +1,109 @@ +import { open } from "node:fs/promises"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { join } from "node:path"; +import { DEFAULT_CHUNK_SIZE_BYTES, EMPTY_SHA256, chunkRecordKey, parseSha256Hex, sha256Hex } from "cultnet-ts/content"; + +/** What the provider reads of a request. The path and the headers are never read: the route is the query alone. */ +export interface ContentChunkRequest { method: string; url: string; } +export interface ContentChunkAnswer { status: number; headers: Readonly>; body?: Uint8Array; } + +/** The port a host implements. Whatever it returns is hashed before a byte is served. `chunkHash` is canonical lowercase hex. */ +export interface ContentChunkStore { readChunk(chunkHash: string, maxBytes: number): Promise; } + +export interface ContentChunkHandlerOptions { + /** Largest chunk served, and the most memory one in-flight request holds. Defaults to the packer's chunk size. */ + maxChunkBytes?: number; + /** Called for every 500 answer; a throwing hook leaves the answer unchanged. */ + onServerFault?: (fault: { chunkHash: string; kind: "hash-mismatch" | "too-large" | "store-error"; error?: unknown }) => void; +} + +/** A store's refusal to read a chunk larger than `maxBytes`. */ +export class ContentChunkTooLargeError extends Error { + constructor(readonly sizeBytes: number) { super("Chunk exceeds the size bound."); } +} + +const errorHeaders = { "Content-Type": "text/plain; charset=utf-8", "X-Content-Type-Options": "nosniff", "Cache-Control": "no-store", "Access-Control-Allow-Origin": "*" }; +const reasons: Record = { 400: "invalid chunk request", 404: "chunk not found", 405: "method not allowed", 500: "chunk unavailable" }; + +function refusal(status: number, extra: Record = {}): ContentChunkAnswer { + const body = new TextEncoder().encode(reasons[status]); + return { status, headers: { ...errorHeaders, ...extra, "Content-Length": String(body.length) }, body }; +} + +const chunkAnswer = (body: Uint8Array): ContentChunkAnswer => ({ + status: 200, + headers: { + "Content-Length": String(body.length), + "Content-Type": "application/octet-stream", + "X-Content-Type-Options": "nosniff", + "Cache-Control": "public, max-age=31536000, immutable, no-transform", + "Access-Control-Allow-Origin": "*", + }, + body, +}); + +/** + * The HTTP answer to `?chunkHash=&recordKey=`, as the C# connector asks it. Every chunk is hashed against its + * name before the first byte is sent, GET and HEAD alike, and nothing is cached between requests. + */ +export function createContentChunkHandler(store: ContentChunkStore, options: ContentChunkHandlerOptions = {}): (request: ContentChunkRequest) => Promise { + const maxChunkBytes = options.maxChunkBytes ?? DEFAULT_CHUNK_SIZE_BYTES; + if (!Number.isInteger(maxChunkBytes) || maxChunkBytes < 1 || maxChunkBytes > 2147483647) throw new RangeError("maxChunkBytes must be an integer in 1..2147483647."); + + const fault = (chunkHash: string, kind: "hash-mismatch" | "too-large" | "store-error", error?: unknown): ContentChunkAnswer => { + try { options.onServerFault?.({ chunkHash, kind, error }); } catch { /* a hook cannot change the answer */ } + return refusal(500); + }; + + const serve = async (url: string): Promise => { + let hex: string; + try { + const query = new URL(url, "http://provider.invalid").searchParams; + const [hash, ...otherHashes] = query.getAll("chunkHash"); + const [key = "", ...otherKeys] = query.getAll("recordKey"); + hex = parseSha256Hex(hash ?? "", "chunkHash"); + if (otherHashes.length || otherKeys.length || (key.trim() !== "" && key !== chunkRecordKey(hex))) return refusal(400); + } catch { return refusal(400); } + if (hex === EMPTY_SHA256) return chunkAnswer(new Uint8Array()); + let bytes: Uint8Array | undefined; + try { bytes = await store.readChunk(hex, maxChunkBytes); } + catch (error) { return fault(hex, error instanceof ContentChunkTooLargeError ? "too-large" : "store-error", error); } + if (!bytes) return refusal(404); + if (bytes.length > maxChunkBytes) return fault(hex, "too-large"); + return await sha256Hex(bytes) === hex ? chunkAnswer(bytes) : fault(hex, "hash-mismatch"); + }; + + return async ({ method, url }) => { + if (method !== "GET" && method !== "HEAD") return refusal(405, { Allow: "GET, HEAD" }); + const { body, ...answer } = await serve(url); + return method === "GET" ? { ...answer, body } : answer; + }; +} + +/** A read-only flat directory: the chunk whose hash is `` lives in `/`. It writes, lists and caches nothing. */ +export function createDirectoryChunkStore(root: string): ContentChunkStore { + return { + async readChunk(chunkHash, maxBytes) { + if (parseSha256Hex(chunkHash) !== chunkHash) throw new Error("chunkHash must be canonical lowercase hex."); + const file = await open(join(root, chunkHash), "r").catch(error => { if (error?.code === "ENOENT") return undefined; throw error; }); + if (!file) return undefined; + try { + const stat = await file.stat(); + if (!stat.isFile()) throw new Error("Chunk is not a regular file."); + if (stat.size > maxBytes) throw new ContentChunkTooLargeError(stat.size); + const bytes = new Uint8Array(stat.size); + const { bytesRead } = await file.read(bytes, 0, stat.size, 0); + return bytes.subarray(0, bytesRead); + } finally { await file.close(); } + }, + }; +} + +/** Mounts a handler on `node:http` or `node:https`: method and URL in, status, headers and body out. */ +export const toNodeListener = (handler: (request: ContentChunkRequest) => Promise) => + (req: IncomingMessage, res: ServerResponse): void => { + handler({ method: req.method ?? "", url: req.url ?? "/" }).catch(() => refusal(500)).then(({ status, headers, body }) => { + res.writeHead(status, headers); + res.end(body); + }); + }; diff --git a/packages/cultmesh-ts/test/content.test.ts b/packages/cultmesh-ts/test/content.test.ts new file mode 100644 index 00000000..7e327c7b --- /dev/null +++ b/packages/cultmesh-ts/test/content.test.ts @@ -0,0 +1,233 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { EMPTY_SHA256, chunkRecordKey, packContent, sha256Hex } from "cultnet-ts/content"; + +import { + ContentChunkTooLargeError, + createContentChunkHandler, + createDirectoryChunkStore, + toNodeListener, + type ContentChunkAnswer, + type ContentChunkHandlerOptions, + type ContentChunkStore, +} from "../src/content"; + +// Chunks come from the packer, so the hashes are the ones a publisher writes. +const packed = async (text: string) => (await packContent("a", new TextEncoder().encode(text))).chunks[0]!; +const decode = (answer: ContentChunkAnswer) => new TextDecoder().decode(answer.body ?? new Uint8Array()); +const OK_HEADERS = { + "Content-Type": "application/octet-stream", + "X-Content-Type-Options": "nosniff", + "Cache-Control": "public, max-age=31536000, immutable, no-transform", + "Access-Control-Allow-Origin": "*", +}; + +function spyStore(chunks: Record): ContentChunkStore & { calls: string[] } { + const calls: string[] = []; + return { calls, readChunk: async hash => { calls.push(hash); return chunks[hash]; } }; +} +const ask = (store: ContentChunkStore, url: string, method = "GET", options?: ContentChunkHandlerOptions) => + createContentChunkHandler(store, options)({ method, url }); + +test("GET of a stored chunk answers 200 with exactly its bytes and the 200 header set", async () => { + const chunk = await packed("hello chunk"); + const answer = await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), `/c?chunkHash=${chunk.chunkHash}`); + assert.equal(answer.status, 200); + assert.deepEqual(answer.body, chunk.payload); + assert.deepEqual(answer.headers, { ...OK_HEADERS, "Content-Length": String(chunk.payload.length) }); +}); + +test("HEAD answers the same status and headers as GET with no body, and HEAD of a corrupt chunk answers 500", async () => { + const chunk = await packed("hello chunk"); + const url = `/c?chunkHash=${chunk.chunkHash}`; + const get = await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url); + const head = await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url, "HEAD"); + assert.equal(head.status, 200); + assert.deepEqual(head.headers, get.headers); + assert.equal(head.body, undefined); + const corrupt = await ask(spyStore({ [chunk.chunkHash]: new TextEncoder().encode("hello chunK") }), url, "HEAD"); + assert.equal(corrupt.status, 500); + assert.equal(corrupt.body, undefined); +}); + +test("chunkHash 'SHA256: ' answers like the canonical hash; recordKey absent, empty or exact answers 200", async () => { + const chunk = await packed("hello chunk"); + const store = spyStore({ [chunk.chunkHash]: chunk.payload }); + const hash = encodeURIComponent(`SHA256:${chunk.chunkHash.toUpperCase()} `); + const keys = ["", "&recordKey=", "&recordKey=%20%20", `&recordKey=${encodeURIComponent(chunkRecordKey(chunk.chunkHash))}`]; + for (const key of keys) { + const answer = await ask(store, `/c?chunkHash=${hash}${key}`); + assert.equal(answer.status, 200, key); + assert.deepEqual(answer.body, chunk.payload); + } + assert.deepEqual(new Set(store.calls), new Set([chunk.chunkHash])); +}); + +test("a recordKey for another hash, a repeated recordKey, a missing or repeated chunkHash, 63 hex, non-hex, '../x' and '%2e%2e%2f' answer 400 and the store is never called", async () => { + const chunk = await packed("hello chunk"); + const other = await packed("another"); + const store = spyStore({ [chunk.chunkHash]: chunk.payload }); + const h = chunk.chunkHash; + const key = encodeURIComponent(chunkRecordKey(h)); + const urls = [ + `/c?chunkHash=${h}&recordKey=${encodeURIComponent(chunkRecordKey(other.chunkHash))}`, + `/c?chunkHash=${h}&recordKey=x`, + `/c?chunkHash=${h}&recordKey=${key}&recordKey=${key}`, + `/c?recordKey=${key}`, + "/c", + `/c?chunkHash=${h}&chunkHash=${h}`, + `/c?chunkHash=${h.slice(1)}`, + `/c?chunkHash=${"g".repeat(64)}`, + "/c?chunkHash=../x", + "/c?chunkHash=%2e%2e%2f", + "/c?chunkHash=", + ]; + for (const url of urls) { + const answer = await ask(store, url); + assert.equal(answer.status, 400, url); + assert.equal(answer.headers["Cache-Control"], "no-store"); + } + assert.deepEqual(store.calls, []); +}); + +test("a valid unknown hash answers 404 with Cache-Control: no-store", async () => { + const chunk = await packed("hello chunk"); + const answer = await ask(spyStore({}), `/c?chunkHash=${chunk.chunkHash}`); + assert.equal(answer.status, 404); + assert.equal(answer.headers["Cache-Control"], "no-store"); + assert.equal(decode(answer), "chunk not found"); +}); + +test("the empty hash answers 200 with Content-Length 0 from a store that throws when called", async () => { + const store: ContentChunkStore = { readChunk: async () => { throw new Error("must not be called"); } }; + const answer = await ask(store, `/c?chunkHash=${EMPTY_SHA256}`); + assert.equal(answer.status, 200); + assert.equal(answer.headers["Content-Length"], "0"); + assert.equal(answer.body?.length, 0); +}); + +test("stored bytes that do not hash to their name answer 500, the body holds none of them, and onServerFault gets hash-mismatch; a throwing hook still yields 500", async () => { + const chunk = await packed("hello chunk"); + const canary = "CANARY-corrupt-bytes"; + const store = spyStore({ [chunk.chunkHash]: new TextEncoder().encode(canary) }); + const faults: unknown[] = []; + const answer = await ask(store, `/c?chunkHash=${chunk.chunkHash}`, "GET", { onServerFault: f => faults.push(f) }); + assert.equal(answer.status, 500); + assert.equal(decode(answer), "chunk unavailable"); + assert.equal(JSON.stringify(answer).includes(canary), false); + assert.deepEqual(faults, [{ chunkHash: chunk.chunkHash, kind: "hash-mismatch", error: undefined }]); + const throwing = await ask(store, `/c?chunkHash=${chunk.chunkHash}`, "GET", { onServerFault: () => { throw new Error("hook"); } }); + assert.equal(throwing.status, 500); +}); + +test("a valid 17-byte chunk with maxChunkBytes 16 answers 500 too-large, a store rejection answers 500 store-error, and maxChunkBytes 0, 2^31 and 1.5 throw RangeError", async () => { + const chunk = await packed("0123456789abcdefg"); + assert.equal(chunk.payload.length, 17); + const kinds: string[] = []; + const options = { maxChunkBytes: 16, onServerFault: (f: { kind: string }) => kinds.push(f.kind) }; + const url = `/c?chunkHash=${chunk.chunkHash}`; + assert.equal((await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url, "GET", options)).status, 500); + const refusing: ContentChunkStore = { readChunk: async () => { throw new ContentChunkTooLargeError(17); } }; + assert.equal((await ask(refusing, url, "GET", options)).status, 500); + const failing: ContentChunkStore = { readChunk: async () => { throw new Error("disk"); } }; + assert.equal((await ask(failing, url, "GET", options)).status, 500); + assert.deepEqual(kinds, ["too-large", "too-large", "store-error"]); + assert.equal((await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url, "GET", { maxChunkBytes: 17 })).status, 200); + for (const bad of [0, 2 ** 31, 1.5, Number.NaN]) assert.throws(() => createContentChunkHandler(spyStore({}), { maxChunkBytes: bad }), RangeError, String(bad)); + assert.doesNotThrow(() => createContentChunkHandler(spyStore({}), { maxChunkBytes: 2 ** 31 - 1 })); +}); + +test("POST, PUT, DELETE and OPTIONS answer 405 with Allow: GET, HEAD", async () => { + const chunk = await packed("hello chunk"); + const store = spyStore({ [chunk.chunkHash]: chunk.payload }); + for (const method of ["POST", "PUT", "DELETE", "OPTIONS", ""]) { + const answer = await ask(store, `/c?chunkHash=${chunk.chunkHash}`, method); + assert.equal(answer.status, 405, method); + assert.equal(answer.headers.Allow, "GET, HEAD"); + assert.equal(answer.headers["Access-Control-Allow-Origin"], "*"); + } + assert.deepEqual(store.calls, []); +}); + +test("'/anything/../etc/passwd?chunkHash=' answers exactly as '/?chunkHash=', and '/' with no query answers 400", async () => { + const chunk = await packed("hello chunk"); + const store = spyStore({ [chunk.chunkHash]: chunk.payload }); + const plain = await ask(store, `/?chunkHash=${chunk.chunkHash}`); + assert.deepEqual(await ask(store, `/anything/../etc/passwd?chunkHash=${chunk.chunkHash}`), plain); + assert.equal((await ask(store, `/${chunk.chunkHash}`)).status, 400); +}); + +test("error answers carry ACAO * and never echo the request's hash or key", async () => { + const canary = "CANARY-request-input"; + const chunk = await packed("hello chunk"); + const urls = [`/c?chunkHash=${canary}`, `/c?chunkHash=${chunk.chunkHash}&recordKey=${canary}`, `/${canary}?chunkHash=${chunk.chunkHash}`]; + for (const url of urls) { + for (const answer of [await ask(spyStore({}), url), await ask(spyStore({}), url, "POST")]) { + assert.equal(answer.headers["Access-Control-Allow-Origin"], "*"); + assert.equal(answer.headers["X-Content-Type-Options"], "nosniff"); + assert.equal(answer.headers["Content-Length"], String(answer.body?.length)); + assert.equal(JSON.stringify(answer).includes(canary), false); + } + } +}); + +test("directory store reads canonical hex and rejects non-canonical names, a directory and an oversized file", async () => { + const root = await mkdtemp(join(tmpdir(), "cm-content-")); + try { + const chunk = await packed("0123456789abcdefg"); + const store = createDirectoryChunkStore(root); + await writeFile(join(root, chunk.chunkHash), chunk.payload); + assert.deepEqual(await store.readChunk(chunk.chunkHash, 17), chunk.payload); + assert.equal(await store.readChunk(await sha256Hex(new Uint8Array([1])), 17), undefined); + // The root does not exist for these, so only the name check can reject them before a filesystem call. + const nowhere = createDirectoryChunkStore(join(root, "missing")); + for (const name of [chunk.chunkHash.toUpperCase(), `sha256:${chunk.chunkHash}`, `../${chunk.chunkHash}`, chunk.chunkHash.slice(1), ` ${chunk.chunkHash}`]) { + await assert.rejects(nowhere.readChunk(name, 17), name); + } + await assert.rejects(store.readChunk(chunk.chunkHash, 16), (error: unknown) => error instanceof ContentChunkTooLargeError && error.sizeBytes === 17); + const dirHash = await sha256Hex(new Uint8Array([2])); + await mkdir(join(root, dirHash)); + await assert.rejects(store.readChunk(dirHash, 17)); + // Through the handler: a file whose bytes do not match its name is a 500, and the right bytes are a 200. + const wrongHash = await sha256Hex(new Uint8Array([3])); + await writeFile(join(root, wrongHash), chunk.payload); + const handler = createContentChunkHandler(store); + assert.equal((await handler({ method: "GET", url: `/?chunkHash=${wrongHash}` })).status, 500); + assert.equal((await handler({ method: "GET", url: `/?chunkHash=${chunk.chunkHash}` })).status, 200); + assert.equal((await handler({ method: "GET", url: `/?chunkHash=${dirHash}` })).status, 500); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("over node:http through toNodeListener: GET with Range and Accept-Encoding: gzip gets 200 with the full identity body; HEAD carries Content-Length; a handler rejection answers 500", async () => { + const chunk = await packed("hello chunk over http"); + const handler = createContentChunkHandler(spyStore({ [chunk.chunkHash]: chunk.payload })); + const serve = async (listener: ReturnType, run: (base: string) => Promise) => { + const server: Server = createServer(listener); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + try { await run(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); } + finally { await new Promise(resolve => { server.close(() => resolve()); server.closeAllConnections(); }); } + }; + await serve(toNodeListener(handler), async base => { + const url = `${base}/mount?chunkHash=${chunk.chunkHash}`; + const get = await fetch(url, { headers: { Range: "bytes=2-5", "Accept-Encoding": "gzip" } }); + assert.equal(get.status, 200); + assert.equal(get.headers.get("content-encoding"), null); + assert.equal(get.headers.get("accept-ranges"), null); + assert.deepEqual(new Uint8Array(await get.arrayBuffer()), chunk.payload); + const head = await fetch(url, { method: "HEAD" }); + assert.equal(head.status, 200); + assert.equal(head.headers.get("content-length"), String(chunk.payload.length)); + assert.equal((await head.arrayBuffer()).byteLength, 0); + }); + await serve(toNodeListener(async () => { throw new Error("boom"); }), async base => { + const response = await fetch(`${base}/`); + assert.equal(response.status, 500); + assert.equal(await response.text(), "chunk unavailable"); + }); +});