GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

cultmesh-ts/content: a chunk handler that verifies before serving, a read-only directory store, a node:http listener

Browse source 
Builds. The ./content subpath is the only way to reach it, so the package root gains no node:fs import.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-19 15:28:02 +02:00
commit 2e9a44925c
3 changed files with 347 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
packages/cultmesh-ts/package.json
View file

@ -9,6 +9,11 @@
"types": "./dist/index.d.ts",
"require": "./dist/index.js",
"default": "./dist/index.js"
},
"./content": {
"types": "./dist/content.d.ts",
"require": "./dist/content.js",
"default": "./dist/content.js"
}
},
"files": [

109
packages/cultmesh-ts/src/content.ts Normal file
View file

@ -0,0 +1,109 @@
import { open } from "node:fs/promises";
import type { IncomingMessage, ServerResponse } from "node:http";
import { join } from "node:path";
import { DEFAULT_CHUNK_SIZE_BYTES, EMPTY_SHA256, chunkRecordKey, parseSha256Hex, sha256Hex } from "cultnet-ts/content";
/** What the provider reads of a request. The path and the headers are never read: the route is the query alone. */
export interface ContentChunkRequest { method: string; url: string; }
export interface ContentChunkAnswer { status: number; headers: Readonly<Record<string, string>>; body?: Uint8Array; }
/** The port a host implements. Whatever it returns is hashed before a byte is served. `chunkHash` is canonical lowercase hex. */
export interface ContentChunkStore { readChunk(chunkHash: string, maxBytes: number): Promise<Uint8Array | undefined>; }
export interface ContentChunkHandlerOptions {
/** Largest chunk served, and the most memory one in-flight request holds. Defaults to the packer's chunk size. */
maxChunkBytes?: number;
/** Called for every 500 answer; a throwing hook leaves the answer unchanged. */
onServerFault?: (fault: { chunkHash: string; kind: "hash-mismatch" | "too-large" | "store-error"; error?: unknown }) => void;
}
/** A store's refusal to read a chunk larger than `maxBytes`. */
export class ContentChunkTooLargeError extends Error {
constructor(readonly sizeBytes: number) { super("Chunk exceeds the size bound."); }
}
const errorHeaders = { "Content-Type": "text/plain; charset=utf-8", "X-Content-Type-Options": "nosniff", "Cache-Control": "no-store", "Access-Control-Allow-Origin": "*" };
const reasons: Record<number, string> = { 400: "invalid chunk request", 404: "chunk not found", 405: "method not allowed", 500: "chunk unavailable" };
function refusal(status: number, extra: Record<string, string> = {}): ContentChunkAnswer {
const body = new TextEncoder().encode(reasons[status]);
return { status, headers: { ...errorHeaders, ...extra, "Content-Length": String(body.length) }, body };
}
const chunkAnswer = (body: Uint8Array): ContentChunkAnswer => ({
status: 200,
headers: {
"Content-Length": String(body.length),
"Content-Type": "application/octet-stream",
"X-Content-Type-Options": "nosniff",
"Cache-Control": "public, max-age=31536000, immutable, no-transform",
"Access-Control-Allow-Origin": "*",
},
body,
});
/**
* The HTTP answer to `?chunkHash=<hex>&recordKey=<key>`, as the C# connector asks it. Every chunk is hashed against its
* name before the first byte is sent, GET and HEAD alike, and nothing is cached between requests.
*/
export function createContentChunkHandler(store: ContentChunkStore, options: ContentChunkHandlerOptions = {}): (request: ContentChunkRequest) => Promise<ContentChunkAnswer> {
const maxChunkBytes = options.maxChunkBytes ?? DEFAULT_CHUNK_SIZE_BYTES;
if (!Number.isInteger(maxChunkBytes) || maxChunkBytes < 1 || maxChunkBytes > 2147483647) throw new RangeError("maxChunkBytes must be an integer in 1..2147483647.");
const fault = (chunkHash: string, kind: "hash-mismatch" | "too-large" | "store-error", error?: unknown): ContentChunkAnswer => {
try { options.onServerFault?.({ chunkHash, kind, error }); } catch { /* a hook cannot change the answer */ }
return refusal(500);
};
const serve = async (url: string): Promise<ContentChunkAnswer> => {
let hex: string;
try {
const query = new URL(url, "http://provider.invalid").searchParams;
const [hash, ...otherHashes] = query.getAll("chunkHash");
const [key = "", ...otherKeys] = query.getAll("recordKey");
hex = parseSha256Hex(hash ?? "", "chunkHash");
if (otherHashes.length || otherKeys.length || (key.trim() !== "" && key !== chunkRecordKey(hex))) return refusal(400);
} catch { return refusal(400); }
if (hex === EMPTY_SHA256) return chunkAnswer(new Uint8Array());
let bytes: Uint8Array | undefined;
try { bytes = await store.readChunk(hex, maxChunkBytes); }
catch (error) { return fault(hex, error instanceof ContentChunkTooLargeError ? "too-large" : "store-error", error); }
if (!bytes) return refusal(404);
if (bytes.length > maxChunkBytes) return fault(hex, "too-large");
return await sha256Hex(bytes) === hex ? chunkAnswer(bytes) : fault(hex, "hash-mismatch");
};
return async ({ method, url }) => {
if (method !== "GET" && method !== "HEAD") return refusal(405, { Allow: "GET, HEAD" });
const { body, ...answer } = await serve(url);
return method === "GET" ? { ...answer, body } : answer;
};
}
/** A read-only flat directory: the chunk whose hash is `<hex>` lives in `<root>/<hex>`. It writes, lists and caches nothing. */
export function createDirectoryChunkStore(root: string): ContentChunkStore {
return {
async readChunk(chunkHash, maxBytes) {
if (parseSha256Hex(chunkHash) !== chunkHash) throw new Error("chunkHash must be canonical lowercase hex.");
const file = await open(join(root, chunkHash), "r").catch(error => { if (error?.code === "ENOENT") return undefined; throw error; });
if (!file) return undefined;
try {
const stat = await file.stat();
if (!stat.isFile()) throw new Error("Chunk is not a regular file.");
if (stat.size > maxBytes) throw new ContentChunkTooLargeError(stat.size);
const bytes = new Uint8Array(stat.size);
const { bytesRead } = await file.read(bytes, 0, stat.size, 0);
return bytes.subarray(0, bytesRead);
} finally { await file.close(); }
},
};
}
/** Mounts a handler on `node:http` or `node:https`: method and URL in, status, headers and body out. */
export const toNodeListener = (handler: (request: ContentChunkRequest) => Promise<ContentChunkAnswer>) =>
(req: IncomingMessage, res: ServerResponse): void => {
handler({ method: req.method ?? "", url: req.url ?? "/" }).catch(() => refusal(500)).then(({ status, headers, body }) => {
res.writeHead(status, headers);
res.end(body);
});
};

233
packages/cultmesh-ts/test/content.test.ts Normal file
View file

@ -0,0 +1,233 @@
import assert from "node:assert/strict";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { EMPTY_SHA256, chunkRecordKey, packContent, sha256Hex } from "cultnet-ts/content";
import {
ContentChunkTooLargeError,
createContentChunkHandler,
createDirectoryChunkStore,
toNodeListener,
type ContentChunkAnswer,
type ContentChunkHandlerOptions,
type ContentChunkStore,
} from "../src/content";
// Chunks come from the packer, so the hashes are the ones a publisher writes.
const packed = async (text: string) => (await packContent("a", new TextEncoder().encode(text))).chunks[0]!;
const decode = (answer: ContentChunkAnswer) => new TextDecoder().decode(answer.body ?? new Uint8Array());
const OK_HEADERS = {
"Content-Type": "application/octet-stream",
"X-Content-Type-Options": "nosniff",
"Cache-Control": "public, max-age=31536000, immutable, no-transform",
"Access-Control-Allow-Origin": "*",
};
function spyStore(chunks: Record<string, Uint8Array>): ContentChunkStore & { calls: string[] } {
const calls: string[] = [];
return { calls, readChunk: async hash => { calls.push(hash); return chunks[hash]; } };
}
const ask = (store: ContentChunkStore, url: string, method = "GET", options?: ContentChunkHandlerOptions) =>
createContentChunkHandler(store, options)({ method, url });
test("GET of a stored chunk answers 200 with exactly its bytes and the 200 header set", async () => {
const chunk = await packed("hello chunk");
const answer = await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), `/c?chunkHash=${chunk.chunkHash}`);
assert.equal(answer.status, 200);
assert.deepEqual(answer.body, chunk.payload);
assert.deepEqual(answer.headers, { ...OK_HEADERS, "Content-Length": String(chunk.payload.length) });
});
test("HEAD answers the same status and headers as GET with no body, and HEAD of a corrupt chunk answers 500", async () => {
const chunk = await packed("hello chunk");
const url = `/c?chunkHash=${chunk.chunkHash}`;
const get = await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url);
const head = await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url, "HEAD");
assert.equal(head.status, 200);
assert.deepEqual(head.headers, get.headers);
assert.equal(head.body, undefined);
const corrupt = await ask(spyStore({ [chunk.chunkHash]: new TextEncoder().encode("hello chunK") }), url, "HEAD");
assert.equal(corrupt.status, 500);
assert.equal(corrupt.body, undefined);
});
test("chunkHash 'SHA256:<UPPER HEX> ' answers like the canonical hash; recordKey absent, empty or exact answers 200", async () => {
const chunk = await packed("hello chunk");
const store = spyStore({ [chunk.chunkHash]: chunk.payload });
const hash = encodeURIComponent(`SHA256:${chunk.chunkHash.toUpperCase()} `);
const keys = ["", "&recordKey=", "&recordKey=%20%20", `&recordKey=${encodeURIComponent(chunkRecordKey(chunk.chunkHash))}`];
for (const key of keys) {
const answer = await ask(store, `/c?chunkHash=${hash}${key}`);
assert.equal(answer.status, 200, key);
assert.deepEqual(answer.body, chunk.payload);
}
assert.deepEqual(new Set(store.calls), new Set([chunk.chunkHash]));
});
test("a recordKey for another hash, a repeated recordKey, a missing or repeated chunkHash, 63 hex, non-hex, '../x' and '%2e%2e%2f' answer 400 and the store is never called", async () => {
const chunk = await packed("hello chunk");
const other = await packed("another");
const store = spyStore({ [chunk.chunkHash]: chunk.payload });
const h = chunk.chunkHash;
const key = encodeURIComponent(chunkRecordKey(h));
const urls = [
`/c?chunkHash=${h}&recordKey=${encodeURIComponent(chunkRecordKey(other.chunkHash))}`,
`/c?chunkHash=${h}&recordKey=x`,
`/c?chunkHash=${h}&recordKey=${key}&recordKey=${key}`,
`/c?recordKey=${key}`,
"/c",
`/c?chunkHash=${h}&chunkHash=${h}`,
`/c?chunkHash=${h.slice(1)}`,
`/c?chunkHash=${"g".repeat(64)}`,
"/c?chunkHash=../x",
"/c?chunkHash=%2e%2e%2f",
"/c?chunkHash=",
];
for (const url of urls) {
const answer = await ask(store, url);
assert.equal(answer.status, 400, url);
assert.equal(answer.headers["Cache-Control"], "no-store");
}
assert.deepEqual(store.calls, []);
});
test("a valid unknown hash answers 404 with Cache-Control: no-store", async () => {
const chunk = await packed("hello chunk");
const answer = await ask(spyStore({}), `/c?chunkHash=${chunk.chunkHash}`);
assert.equal(answer.status, 404);
assert.equal(answer.headers["Cache-Control"], "no-store");
assert.equal(decode(answer), "chunk not found");
});
test("the empty hash answers 200 with Content-Length 0 from a store that throws when called", async () => {
const store: ContentChunkStore = { readChunk: async () => { throw new Error("must not be called"); } };
const answer = await ask(store, `/c?chunkHash=${EMPTY_SHA256}`);
assert.equal(answer.status, 200);
assert.equal(answer.headers["Content-Length"], "0");
assert.equal(answer.body?.length, 0);
});
test("stored bytes that do not hash to their name answer 500, the body holds none of them, and onServerFault gets hash-mismatch; a throwing hook still yields 500", async () => {
const chunk = await packed("hello chunk");
const canary = "CANARY-corrupt-bytes";
const store = spyStore({ [chunk.chunkHash]: new TextEncoder().encode(canary) });
const faults: unknown[] = [];
const answer = await ask(store, `/c?chunkHash=${chunk.chunkHash}`, "GET", { onServerFault: f => faults.push(f) });
assert.equal(answer.status, 500);
assert.equal(decode(answer), "chunk unavailable");
assert.equal(JSON.stringify(answer).includes(canary), false);
assert.deepEqual(faults, [{ chunkHash: chunk.chunkHash, kind: "hash-mismatch", error: undefined }]);
const throwing = await ask(store, `/c?chunkHash=${chunk.chunkHash}`, "GET", { onServerFault: () => { throw new Error("hook"); } });
assert.equal(throwing.status, 500);
});
test("a valid 17-byte chunk with maxChunkBytes 16 answers 500 too-large, a store rejection answers 500 store-error, and maxChunkBytes 0, 2^31 and 1.5 throw RangeError", async () => {
const chunk = await packed("0123456789abcdefg");
assert.equal(chunk.payload.length, 17);
const kinds: string[] = [];
const options = { maxChunkBytes: 16, onServerFault: (f: { kind: string }) => kinds.push(f.kind) };
const url = `/c?chunkHash=${chunk.chunkHash}`;
assert.equal((await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url, "GET", options)).status, 500);
const refusing: ContentChunkStore = { readChunk: async () => { throw new ContentChunkTooLargeError(17); } };
assert.equal((await ask(refusing, url, "GET", options)).status, 500);
const failing: ContentChunkStore = { readChunk: async () => { throw new Error("disk"); } };
assert.equal((await ask(failing, url, "GET", options)).status, 500);
assert.deepEqual(kinds, ["too-large", "too-large", "store-error"]);
assert.equal((await ask(spyStore({ [chunk.chunkHash]: chunk.payload }), url, "GET", { maxChunkBytes: 17 })).status, 200);
for (const bad of [0, 2 ** 31, 1.5, Number.NaN]) assert.throws(() => createContentChunkHandler(spyStore({}), { maxChunkBytes: bad }), RangeError, String(bad));
assert.doesNotThrow(() => createContentChunkHandler(spyStore({}), { maxChunkBytes: 2 ** 31 - 1 }));
});
test("POST, PUT, DELETE and OPTIONS answer 405 with Allow: GET, HEAD", async () => {
const chunk = await packed("hello chunk");
const store = spyStore({ [chunk.chunkHash]: chunk.payload });
for (const method of ["POST", "PUT", "DELETE", "OPTIONS", ""]) {
const answer = await ask(store, `/c?chunkHash=${chunk.chunkHash}`, method);
assert.equal(answer.status, 405, method);
assert.equal(answer.headers.Allow, "GET, HEAD");
assert.equal(answer.headers["Access-Control-Allow-Origin"], "*");
}
assert.deepEqual(store.calls, []);
});
test("'/anything/../etc/passwd?chunkHash=<h>' answers exactly as '/?chunkHash=<h>', and '/<h>' with no query answers 400", async () => {
const chunk = await packed("hello chunk");
const store = spyStore({ [chunk.chunkHash]: chunk.payload });
const plain = await ask(store, `/?chunkHash=${chunk.chunkHash}`);
assert.deepEqual(await ask(store, `/anything/../etc/passwd?chunkHash=${chunk.chunkHash}`), plain);
assert.equal((await ask(store, `/${chunk.chunkHash}`)).status, 400);
});
test("error answers carry ACAO * and never echo the request's hash or key", async () => {
const canary = "CANARY-request-input";
const chunk = await packed("hello chunk");
const urls = [`/c?chunkHash=${canary}`, `/c?chunkHash=${chunk.chunkHash}&recordKey=${canary}`, `/${canary}?chunkHash=${chunk.chunkHash}`];
for (const url of urls) {
for (const answer of [await ask(spyStore({}), url), await ask(spyStore({}), url, "POST")]) {
assert.equal(answer.headers["Access-Control-Allow-Origin"], "*");
assert.equal(answer.headers["X-Content-Type-Options"], "nosniff");
assert.equal(answer.headers["Content-Length"], String(answer.body?.length));
assert.equal(JSON.stringify(answer).includes(canary), false);
}
}
});
test("directory store reads canonical hex and rejects non-canonical names, a directory and an oversized file", async () => {
const root = await mkdtemp(join(tmpdir(), "cm-content-"));
try {
const chunk = await packed("0123456789abcdefg");
const store = createDirectoryChunkStore(root);
await writeFile(join(root, chunk.chunkHash), chunk.payload);
assert.deepEqual(await store.readChunk(chunk.chunkHash, 17), chunk.payload);
assert.equal(await store.readChunk(await sha256Hex(new Uint8Array([1])), 17), undefined);
// The root does not exist for these, so only the name check can reject them before a filesystem call.
const nowhere = createDirectoryChunkStore(join(root, "missing"));
for (const name of [chunk.chunkHash.toUpperCase(), `sha256:${chunk.chunkHash}`, `../${chunk.chunkHash}`, chunk.chunkHash.slice(1), ` ${chunk.chunkHash}`]) {
await assert.rejects(nowhere.readChunk(name, 17), name);
}
await assert.rejects(store.readChunk(chunk.chunkHash, 16), (error: unknown) => error instanceof ContentChunkTooLargeError && error.sizeBytes === 17);
const dirHash = await sha256Hex(new Uint8Array([2]));
await mkdir(join(root, dirHash));
await assert.rejects(store.readChunk(dirHash, 17));
// Through the handler: a file whose bytes do not match its name is a 500, and the right bytes are a 200.
const wrongHash = await sha256Hex(new Uint8Array([3]));
await writeFile(join(root, wrongHash), chunk.payload);
const handler = createContentChunkHandler(store);
assert.equal((await handler({ method: "GET", url: `/?chunkHash=${wrongHash}` })).status, 500);
assert.equal((await handler({ method: "GET", url: `/?chunkHash=${chunk.chunkHash}` })).status, 200);
assert.equal((await handler({ method: "GET", url: `/?chunkHash=${dirHash}` })).status, 500);
} finally { await rm(root, { recursive: true, force: true }); }
});
test("over node:http through toNodeListener: GET with Range and Accept-Encoding: gzip gets 200 with the full identity body; HEAD carries Content-Length; a handler rejection answers 500", async () => {
const chunk = await packed("hello chunk over http");
const handler = createContentChunkHandler(spyStore({ [chunk.chunkHash]: chunk.payload }));
const serve = async (listener: ReturnType<typeof toNodeListener>, run: (base: string) => Promise<void>) => {
const server: Server = createServer(listener);
await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
try { await run(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); }
finally { await new Promise<void>(resolve => { server.close(() => resolve()); server.closeAllConnections(); }); }
};
await serve(toNodeListener(handler), async base => {
const url = `${base}/mount?chunkHash=${chunk.chunkHash}`;
const get = await fetch(url, { headers: { Range: "bytes=2-5", "Accept-Encoding": "gzip" } });
assert.equal(get.status, 200);
assert.equal(get.headers.get("content-encoding"), null);
assert.equal(get.headers.get("accept-ranges"), null);
assert.deepEqual(new Uint8Array(await get.arrayBuffer()), chunk.payload);
const head = await fetch(url, { method: "HEAD" });
assert.equal(head.status, 200);
assert.equal(head.headers.get("content-length"), String(chunk.payload.length));
assert.equal((await head.arrayBuffer()).byteLength, 0);
});
await serve(toNodeListener(async () => { throw new Error("boom"); }), async base => {
const response = await fetch(`${base}/`);
assert.equal(response.status, 500);
assert.equal(await response.text(), "chunk unavailable");
});
});