diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index 086e54bd..f53aae75 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -103,7 +103,8 @@ jobs: # Linux at all, and the browser client's never ran in CI on any OS. # packages/cultnet-ts verifies both vector files (content.test.ts judges both # C# content files, the Rust one and its own); cultmesh-browser proves - # the browser client reads the shared rules rather than its own. + # the browser client reads the shared rules rather than its own, and that its content reader releases only verified + # bytes, requesting the same URLs as the C# HTTPS connector. # No workflow ran cultcache-ts's tests, and its test script listed two files by hand while a third sat compiled # and unrun. The wiring check fails when any TypeScript test file is run by no script, and the cache's own suite # runs here beside the packages built on it. diff --git a/packages/cultmesh-browser/README.md b/packages/cultmesh-browser/README.md index e130171c..52410834 100644 --- a/packages/cultmesh-browser/README.md +++ b/packages/cultmesh-browser/README.md @@ -69,3 +69,31 @@ Run `scripts/verify-eve-browser-network.mjs --eve-root ` for the real Chromium + C# headless chronology. The witness publishes a lower-priority decoy endpoint owned by another runtime and proves that neither client connects to it before and after route rotation. + +## Verified content + +`fetchVerifiedContent(endpoint, manifest)` and +`fetchVerifiedChunk(endpoint, hash, size)` read CultMesh CDN content over HTTP. +The endpoint comes from CultMesh and never from content: a manifest carries no +URL. No byte is released that failed its chunk or whole-body SHA-256 (WebCrypto, +so a secure context in browsers). Only https is accepted, and http only on +loopback; a redirect is a `network` failure, and an endpoint spelling on which +.NET `UriBuilder` and WHATWG `URL` would build different request URLs (non-ASCII, +`'` `{` `}` `|` `^` `\` or a space, a stray `%`, an escaped unreserved character) +is `insecure-endpoint`. A size the runtime cannot allocate is `too-large`. Options: `maxConcurrentChunkRequests` (1 to 32, default 4), `maxBytes` +(default 256 MiB, checked before any request), `signal` (an abort rejects with +the signal's own reason). Failures are `CultMeshContentError` with a `code` +(`manifest-invalid`, `too-large`, `insecure-endpoint`, `webcrypto-unavailable`, +`network`, `http-status`, `length-mismatch`, `chunk-hash-mismatch`, +`content-hash-mismatch`); messages never carry the endpoint or body bytes. The +byte count actually received is judged, never a length header, because Fetch +decodes content codings. + +```ts +import { fetchVerifiedContent } from "cultmesh-browser"; + +const bytes = await fetchVerifiedContent(endpoint, manifest, { + signal: AbortSignal.timeout(30_000), + maxConcurrentChunkRequests: 8, +}); +``` diff --git a/packages/cultmesh-browser/src/content.ts b/packages/cultmesh-browser/src/content.ts new file mode 100644 index 00000000..c5f95a04 --- /dev/null +++ b/packages/cultmesh-browser/src/content.ts @@ -0,0 +1,156 @@ +// Verified reads of CultMesh CDN content over HTTP(S). This module is the only place bytes are released to a caller, and it +// releases them after judging what it received: each chunk against its hash, then the whole body against the manifest's. +// The response's declared length and coding are never consulted (Fetch decodes codings; the bytes received are the evidence). +// Hashing, hex rules, record keys, the loopback rule and manifest validation belong to cultnet-ts and are only called here. + +import { EMPTY_SHA256, chunkRecordKey, parseSha256Hex, sha256Hex, validateManifest, type CultMeshCdnArtifactManifest } from "cultnet-ts/content"; +import { isLoopbackEndpoint } from "cultnet-ts/authority"; + +export type CultMeshContentErrorCode = + | "manifest-invalid" | "too-large" | "insecure-endpoint" | "webcrypto-unavailable" | "network" + | "http-status" | "length-mismatch" | "chunk-hash-mismatch" | "content-hash-mismatch"; + +const MESSAGES: Record = { + "manifest-invalid": "The CDN manifest or chunk description is invalid.", + "too-large": "The content is larger than maxBytes.", + "insecure-endpoint": "Content endpoints must be plain https, or http on loopback.", + "webcrypto-unavailable": "WebCrypto SHA-256 is unavailable.", + "network": "The content request failed.", + "http-status": "The content server did not answer 200.", + "length-mismatch": "The content body did not match the chunk size.", + "chunk-hash-mismatch": "A content chunk did not match its hash.", + "content-hash-mismatch": "The content did not match the manifest content hash.", +}; + +/** A refusal to release bytes. The message is fixed per code and never carries the endpoint or any body byte. */ +export class CultMeshContentError extends Error { + constructor(readonly code: CultMeshContentErrorCode, readonly chunkHash?: string, readonly status?: number, cause?: unknown) { + super(MESSAGES[code], cause === undefined ? undefined : { cause }); + this.name = "CultMeshContentError"; + } +} + +export interface CultMeshContentFetchOptions { + readonly signal?: AbortSignal; + /** Largest whole body, checked before any request. Default 256 MiB. */ + readonly maxBytes?: number; + /** Chunk requests in flight at once, an integer 1..32. Default 4. */ + readonly maxConcurrentChunkRequests?: number; + readonly fetch?: typeof globalThis.fetch; +} + +interface Resolved { readonly doFetch: typeof globalThis.fetch; readonly caller?: AbortSignal; readonly maxBytes: number; readonly concurrency: number } +interface Job { readonly url: string; readonly hash: string; readonly slot: Uint8Array } + +const fail = (code: CultMeshContentErrorCode, chunkHash?: string, status?: number, cause?: unknown) => new CultMeshContentError(code, chunkHash, status, cause); +const isByteCount = (value: number): boolean => Number.isSafeInteger(value) && value >= 0; + +// Uri.EscapeDataString: encodeURIComponent, plus the five characters it leaves bare. +const escapeData = (value: string): string => encodeURIComponent(value).replace(/[!'()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`); + +// An endpoint the request URL of which is the same string in .NET `UriBuilder` and WHATWG `URL`. Both are free to rewrite what +// they dislike (`'` in a query, `{}|^\` and non-ASCII anywhere, a percent sign that is not an escape, an escaped unreserved +// character, an IDN host), and they rewrite it differently, which would split CDN cache entries between runtimes; so those +// spellings are refused instead of translated. +const PLAIN_ENDPOINT = /^(?:[A-Za-z0-9\-._~:/?#@!$&*()+,;=[\]]|%[0-9A-Fa-f]{2})*$/; +const ESCAPED_UNRESERVED = /%(?:2[de]|3\d|4[1-9a-f]|5[0-9af]|6[1-9a-f]|7[0-9ae])/i; +const isPlainEndpoint = (endpoint: string): boolean => PLAIN_ENDPOINT.test(endpoint) && !ESCAPED_UNRESERVED.test(endpoint); + +/** `CultMeshHttpsContentTransportConnector.BuildChunkUri`: an existing query (even an empty one, leading `?`s trimmed) is kept and joined with `&`. */ +function chunkUrl(endpoint: string, hash: string, recordKey: string): string { + const url = new URL(endpoint); + const kept = url.href.split("#")[0]!.includes("?") ? `${url.search.replace(/^\?+/, "")}&` : ""; + url.search = `${kept}chunkHash=${escapeData(hash)}&recordKey=${escapeData(recordKey)}`; + return url.href; +} + +/** Options, caller abort and endpoint scheme: everything that can be refused without reading the manifest. */ +function resolve(endpoint: string, options: CultMeshContentFetchOptions): Resolved { + const { signal, maxBytes = 268435456, maxConcurrentChunkRequests: concurrency = 4, fetch } = options; + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32) throw new RangeError("maxConcurrentChunkRequests must be an integer from 1 to 32."); + if (!isByteCount(maxBytes)) throw new RangeError("maxBytes must be a non-negative safe integer."); + if (signal?.aborted) throw signal.reason; + const secure = (() => { try { const { protocol } = new URL(endpoint); return isPlainEndpoint(endpoint) && (protocol === "https:" || (protocol === "http:" && isLoopbackEndpoint(endpoint))); } catch { return false; } })(); + if (!secure) throw fail("insecure-endpoint"); + return { doFetch: fetch ?? ((input, init) => globalThis.fetch(input, init)), caller: signal, maxBytes, concurrency }; +} + +/** The one private buffer. A size the runtime cannot allocate is an artifact too large for this reader, not a bad option. */ +function allocate(sizeBytes: number): Uint8Array { + try { return new Uint8Array(sizeBytes); } catch (cause) { throw fail("too-large", undefined, undefined, cause); } +} + +const requireWebCrypto = (): void => { if (!globalThis.crypto?.subtle) throw fail("webcrypto-unavailable"); }; + +/** One chunk: GET (a redirect is a failure: the endpoint gate cannot see where it would lead), status 200, the body streamed into its slot and cut at the first byte past the slot, then the chunk hash. */ +async function readChunk(r: Resolved, signal: AbortSignal, { url, hash, slot }: Job): Promise { + if (slot.length === 0) { if (hash !== EMPTY_SHA256) throw fail("chunk-hash-mismatch", hash); return; } + r.caller?.throwIfAborted(); + const network = (cause: unknown) => r.caller?.aborted ? r.caller.reason : fail("network", hash, undefined, cause); + const response = await r.doFetch(url, { method: "GET", credentials: "omit", redirect: "error", signal }).catch(cause => { throw network(cause); }); + if (response.status !== 200) { void response.body?.cancel().catch(() => {}); throw fail("http-status", hash, response.status); } + const reader = response.body?.getReader(); + let filled = 0; + try { + for (let part = await reader?.read(); part && !part.done; part = await reader!.read()) { + if (filled + part.value.length > slot.length) { void reader!.cancel().catch(() => {}); throw fail("length-mismatch", hash); } + slot.set(part.value, filled); + filled += part.value.length; + } + } catch (error) { throw error instanceof CultMeshContentError ? error : network(error); } + if (filled < slot.length) throw fail("length-mismatch", hash); + if (await sha256Hex(slot) !== hash) throw fail("chunk-hash-mismatch", hash); +} + +/** A sliding window of chunk reads. The first failure aborts every request still in flight and is thrown at once. */ +async function readChunks(r: Resolved, jobs: readonly Job[]): Promise { + const internal = new AbortController(); + const relay = () => internal.abort(r.caller!.reason); + r.caller?.addEventListener("abort", relay); + let next = 0; + const worker = async () => { + for (let job = jobs[next++]; job && !internal.signal.aborted; job = jobs[next++]) { + try { await readChunk(r, internal.signal, job); } catch (error) { internal.abort(error); } + } + }; + // The rejection waits for no worker: a fetch that ignores its signal must not hold a failure back. + const failed = new Promise((_, reject) => internal.signal.addEventListener("abort", () => reject(internal.signal.reason))); + try { await Promise.race([Promise.all(Array.from({ length: Math.min(r.concurrency, jobs.length) }, worker)), failed]); } + finally { r.caller?.removeEventListener("abort", relay); } +} + +/** The whole body of a manifest's artifact, returned only after every chunk and the whole-body hash verified. */ +export async function fetchVerifiedContent(endpoint: string, manifest: CultMeshCdnArtifactManifest, options: CultMeshContentFetchOptions = {}): Promise { + const r = resolve(endpoint, options); + let contentHash: string; + let planned: { hash: string; url: string; offset: number; size: number }[]; + try { + validateManifest(manifest); + contentHash = parseSha256Hex(manifest.contentHash, "contentHash"); + const refs = manifest.chunks; + if (refs.length > 1 && refs.some(ref => ref.sizeBytes === 0)) throw new Error("A zero-size chunk must be the only chunk."); + planned = refs.map(ref => { const hash = parseSha256Hex(ref.chunkHash, "chunkHash"); return { hash, url: chunkUrl(endpoint, hash, ref.recordKey ?? ""), offset: ref.offset, size: ref.sizeBytes }; }); + } catch (cause) { throw fail("manifest-invalid", undefined, undefined, cause); } + if (manifest.sizeBytes > r.maxBytes) throw fail("too-large"); + requireWebCrypto(); + const buffer = allocate(manifest.sizeBytes); + await readChunks(r, planned.map(({ hash, url, offset, size }) => ({ url, hash, slot: buffer.subarray(offset, offset + size) }))); + if (await sha256Hex(buffer) !== contentHash) throw fail("content-hash-mismatch"); + r.caller?.throwIfAborted(); + return buffer; +} + +/** One chunk by hash and size, over the same primitive; the record key is the one a packed manifest carries. */ +export async function fetchVerifiedChunk(endpoint: string, chunkHash: string, sizeBytes: number, options: CultMeshContentFetchOptions = {}): Promise { + const r = resolve(endpoint, options); + let hash: string; + try { + hash = parseSha256Hex(chunkHash, "chunkHash"); + if (!isByteCount(sizeBytes)) throw new Error("Size must be a safe integer."); + } catch (cause) { throw fail("manifest-invalid", undefined, undefined, cause); } + if (sizeBytes > r.maxBytes) throw fail("too-large"); + requireWebCrypto(); + const slot = allocate(sizeBytes); + await readChunks(r, [{ url: chunkUrl(endpoint, hash, chunkRecordKey(hash)), hash, slot }]); + return slot; +} diff --git a/packages/cultmesh-browser/src/index.ts b/packages/cultmesh-browser/src/index.ts index 5c20806c..beae9e36 100644 --- a/packages/cultmesh-browser/src/index.ts +++ b/packages/cultmesh-browser/src/index.ts @@ -32,6 +32,7 @@ import { type CultMeshAuthorityTrustPolicy, type CultMeshP256PublicKey, } from "cultnet-ts/authority"; +export * from "./content.js"; export type CultMeshBrowserIdentity = CultMeshAuthorityIdentity; export type CultMeshBrowserRoute = CultMeshAuthorityRouteView; diff --git a/packages/cultmesh-browser/test/content.test.ts b/packages/cultmesh-browser/test/content.test.ts new file mode 100644 index 00000000..5af89c10 --- /dev/null +++ b/packages/cultmesh-browser/test/content.test.ts @@ -0,0 +1,629 @@ +// The verified content reader, rule by rule. Chunks and manifests come from cultnet-ts's own packContent, so the reader is +// judged against the shapes the provider will serve; the URL expectations are the .NET `BuildChunkUri` AbsoluteUri strings +// probed for the map (F5, F19), not a spelling of this module's own builder. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { gzipSync } from "node:zlib"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "esbuild"; +import { EMPTY_SHA256, packContent, sha256Hex, type CultMeshCdnArtifactManifest } from "cultnet-ts/content"; + +import { CultMeshContentError, fetchVerifiedChunk, fetchVerifiedContent, type CultMeshContentErrorCode } from "../src/content.js"; + +const ENDPOINT = "https://cdn.example/content"; +const body = (length: number): Uint8Array => Uint8Array.from({ length }, (_, index) => (index * 7 + 3) & 255); +const pack = (length: number, chunkSizeBytes: number) => packContent("art", body(length), { chunkSizeBytes, createdAtUtc: "2026-01-01T00:00:00.0000000+00:00" }); +const streamOf = (bytes: Uint8Array, onCancel?: () => void) => + new ReadableStream({ start(controller) { controller.enqueue(bytes); controller.close(); }, cancel: onCancel }); +const withChunk = (manifest: CultMeshCdnArtifactManifest, index: number, change: object): CultMeshCdnArtifactManifest => + ({ ...manifest, chunks: manifest.chunks.map((chunk, i) => i === index ? { ...chunk, ...change } : chunk) }); +const codeOf = (code: CultMeshContentErrorCode, extra: object = {}) => (error: unknown) => + error instanceof CultMeshContentError && error.code === code && Object.entries(extra).every(([key, value]) => (error as unknown as Record)[key] === value); + +/** A manifest-invalid rejection whose kept cause has a message matching `pattern`. */ +const causeIs = (pattern: RegExp) => (error: unknown) => { + const cause = (error as Error).cause; + return codeOf("manifest-invalid")(error) && cause instanceof Error && pattern.test(cause.message); +}; + +/** Fails a stalled promise with a timeout instead of letting the test runner wait for it forever. */ +const within = (promise: Promise, ms = 3000): Promise => { + let timer: NodeJS.Timeout; + const late = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`did not settle within ${ms} ms`)), ms); }); + return Promise.race([promise, late]).finally(() => clearTimeout(timer)); +}; + +interface Call { url: string; init: RequestInit } +type Handler = (call: Call, index: number) => Response | Promise; + +/** A fetch double: records every call, counts requests in flight (the handler's lifetime) and their peak. */ +function double(handler: Handler) { + const calls: Call[] = []; + const state = { inflight: 0, peak: 0 }; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const call = { url: String(input), init: init ?? {} }; + const index = calls.push(call) - 1; + state.peak = Math.max(state.peak, ++state.inflight); + try { return await handler(call, index); } finally { state.inflight--; } + }) as typeof globalThis.fetch; + return { fetch, calls, state }; +} + +/** Serves each requested chunk by the `chunkHash` query parameter, after `delayMs`. */ +function chunkServer(chunks: { chunkHash: string; payload: Uint8Array }[], delayMs = 0) { + const byHash = new Map(chunks.map(chunk => [chunk.chunkHash, chunk.payload])); + return double(async call => { + if (delayMs) await new Promise(resolve => setTimeout(resolve, delayMs)); + const payload = byHash.get(new URL(call.url).searchParams.get("chunkHash")!); + return payload ? new Response(payload as BodyInit) : new Response(null, { status: 404 }); + }); +} + +/** A response that never settles until the request's signal aborts, as a real fetch rejects with the signal's reason. */ +const hangUntilAborted = (call: Call) => new Promise((_, reject) => { + const signal = call.init.signal!; + if (signal.aborted) reject(signal.reason); else signal.addEventListener("abort", () => reject(signal.reason), { once: true }); +}); + +test("request URLs equal the .NET BuildChunkUri AbsoluteUri for every probed endpoint shape", async () => { + const { manifest, chunks } = await pack(1, 4); + const hex = manifest.chunks[0]!.chunkHash!; + const cases: [string, string, string][] = [ + ["https://ygg.example/content", "mesh:cdn:chunk:" + hex, `https://ygg.example/content?chunkHash=${hex}&recordKey=mesh%3Acdn%3Achunk%3A${hex}`], + ["https://ygg.example/content?tenant=sp", "k", `https://ygg.example/content?tenant=sp&chunkHash=${hex}&recordKey=k`], + ["https://h", "k", `https://h/?chunkHash=${hex}&recordKey=k`], + ["https://h/", "k", `https://h/?chunkHash=${hex}&recordKey=k`], + ["https://h:443/x", "k", `https://h/x?chunkHash=${hex}&recordKey=k`], + ["https://H.Example/X/", "k", `https://h.example/X/?chunkHash=${hex}&recordKey=k`], + ["https://h/x#frag", "k", `https://h/x?chunkHash=${hex}&recordKey=k#frag`], + ["https://h/x?a=%20b+c", "k", `https://h/x?a=%20b+c&chunkHash=${hex}&recordKey=k`], + ["https://h/x?", "k", `https://h/x?&chunkHash=${hex}&recordKey=k`], + ["https://h/x??a=1", "k", `https://h/x?a=1&chunkHash=${hex}&recordKey=k`], + ["https://h/x???a=1", "k", `https://h/x?a=1&chunkHash=${hex}&recordKey=k`], + ["https://h/x?a=?b", "k", `https://h/x?a=?b&chunkHash=${hex}&recordKey=k`], + ["https://[::1]:8443/c", "k", `https://[::1]:8443/c?chunkHash=${hex}&recordKey=k`], + ["https://h/x", "", `https://h/x?chunkHash=${hex}&recordKey=`], + ["https://h/x", "k with space/ü&=?#", `https://h/x?chunkHash=${hex}&recordKey=k%20with%20space%2F%C3%BC%26%3D%3F%23`], + ["https://h/x", "!'()*~-._ +é𐐀", `https://h/x?chunkHash=${hex}&recordKey=%21%27%28%29%2A~-._%20%2B%C3%A9%F0%90%90%80`], + ]; + for (const [endpoint, recordKey, expected] of cases) { + const server = chunkServer(chunks); + await fetchVerifiedContent(endpoint, withChunk(manifest, 0, { recordKey }), { fetch: server.fetch }); + assert.equal(server.calls[0]!.url, expected, endpoint); + } + const absent = chunkServer(chunks); + await fetchVerifiedContent(ENDPOINT, withChunk(manifest, 0, { recordKey: undefined }), { fetch: absent.fetch }); + assert.equal(absent.calls[0]!.url, `${ENDPOINT}?chunkHash=${hex}&recordKey=`); +}); + +// Rows probed with .NET 10 UriBuilder (the BuildChunkUri body) at the same endpoints: each accepted endpoint yields exactly the +// AbsoluteUri .NET yields ( stands for the chunkHash and recordKey query tail); each refused endpoint is a spelling on which +// .NET and WHATWG URL disagree (or .NET throws), so the reader refuses it rather than request a different URL. +const PARITY_ACCEPTED: [string, string][] = [ + ["https://h/a%2fb", "https://h/a%2fb?"], + ["https://h/./a/../b", "https://h/b?"], + ["https://h/x?a=b#f?g", "https://h/x?a=b&#f?g"], + ["https://h/x?#f", "https://h/x?&#f"], + ["https://h:0443/x", "https://h/x?"], + ["https://h./x", "https://h./x?"], + ["https://h/x?a=[b]", "https://h/x?a=[b]&"], + ["https://u:p@h/x", "https://u:p@h/x?"], + ["https://h/x?a=1&&", "https://h/x?a=1&&&"], + ["https://h/x?%3F", "https://h/x?%3F&"], + ["https://h:/x", "https://h/x?"], + ["https://h/x?a=b?c", "https://h/x?a=b?c&"], + ["https://h/a;b=c,d", "https://h/a;b=c,d?"], + ["https://h/a!$&()*+,;=:@b", "https://h/a!$&()*+,;=:@b?"], + ["https://h/x?a!$()*+,;=:@/?", "https://h/x?a!$()*+,;=:@/?&"], + ["https://h//x", "https://h//x?"], + ["https://h/x/", "https://h/x/?"], + ["https://H:443", "https://h/?"], + ["https://1.2.3.4/x", "https://1.2.3.4/x?"], + ["https://0x7f.1/x", "https://127.0.0.1/x?"], + ["https://127.1/", "https://127.0.0.1/?"], + ["https://h/x#", "https://h/x?#"], + ["https://h/x?a#", "https://h/x?a&#"], + ["https://h/x?a=1#f", "https://h/x?a=1&#f"], + ["https://h/a/./b", "https://h/a/b?"], + ["https://h/a/../../b", "https://h/b?"], + ["https://h/%2F", "https://h/%2F?"], + ["https://h/a%20b", "https://h/a%20b?"], + ["https://h/a%25b", "https://h/a%25b?"], + ["https://h/x?a=%25", "https://h/x?a=%25&"], + ["https://h/x?a=%2f", "https://h/x?a=%2f&"], + ["https://[::1]/x", "https://[::1]/x?"], + ["https://[2001:DB8::1]:8443/x", "https://[2001:db8::1]:8443/x?"], + ["https://h/x?a=b%3Dc", "https://h/x?a=b%3Dc&"], + ["https://h/-._~", "https://h/-._~?"], + ["https://h/x?=", "https://h/x?=&"], + ["https://h/x?&", "https://h/x?&&"], + ["https://h/x?a&&b", "https://h/x?a&&b&"], + ["http://127.0.0.1:80/x", "http://127.0.0.1/x?"], + ["http://127.0.0.1:8080", "http://127.0.0.1:8080/?"], + ["https://h/%40/x", "https://h/%40/x?"], + ["https://h/x?a=%5B%5D", "https://h/x?a=%5B%5D&"], + ["https://h/%60", "https://h/%60?"], + ["https://h/x?a=%7B%7D", "https://h/x?a=%7B%7D&"], + ["https://h/x?a=%2F%3A%2C", "https://h/x?a=%2F%3A%2C&"], + ["https://h/x?a=%2c%5b%7b", "https://h/x?a=%2c%5b%7b&"], +]; +const PARITY_REFUSED: string[] = [ + "https://h/x?a='b", + "https://h/x?a={b}|^", + "https://h/x?a=\"<>`", + "https://h/x?a=b c", + "https://h/x?a=%7e", + "https://h/x?a=é", + "https://h/{x}", + "https://h/a|b", + "https://h/a^b", + "https://h/a'b", + "https://ex%41mple.com/", + "https://Bücher.de/", + "https://h/%7e", + "https://h/x?a=%zz", + "https://h/x?a=\\b", + "https://h/a\\b", + "https://h/x? ", + "https://h:99999/x", + "https://h/%41", + "https://h/x#a b", + "https://h/x#é", + "https://h/%50", + "https://h/x?a=%5A", + "https://h/x?a=%5f", + "https://h/x?a=%61", + "https://h/%6F", + "https://h/%7a", + "https://h/x?a=%7E", + "https://h/%30", + "https://h/x?a=%39", + "https://h/%2d", + "https://h/x?a=%2E", + "https://h/x?a=%4z", + "https://h/%4", + "https://h/x?%", +]; + +test("request URLs equal .NET UriBuilder's for every plain endpoint spelling, and endpoints on which the two disagree are refused", async () => { + const { manifest, chunks } = await pack(1, 4); + const hex = manifest.chunks[0]!.chunkHash!; + const tail = `chunkHash=${hex}&recordKey=k`; + for (const [endpoint, expected] of PARITY_ACCEPTED) { + const server = chunkServer(chunks); + await fetchVerifiedContent(endpoint, withChunk(manifest, 0, { recordKey: "k" }), { fetch: server.fetch }); + assert.equal(server.calls[0]!.url, expected.replace("", tail), endpoint); + } + for (const endpoint of [...PARITY_REFUSED, "https://h/%", "https://h/%g0", "https://h/x?a=%a", "https://h/x?a=%%41"]) { + const server = chunkServer(chunks); + await assert.rejects(fetchVerifiedContent(endpoint, manifest, { fetch: server.fetch }), codeOf("insecure-endpoint"), endpoint); + await assert.rejects(fetchVerifiedChunk(endpoint, hex, 1, { fetch: server.fetch }), codeOf("insecure-endpoint"), endpoint); + assert.equal(server.calls.length, 0, endpoint); + } +}); + +test("hash spellings are normalized into the URL: a prefixed upper-case manifest hash is requested as bare lower-case hex", async () => { + const { manifest, chunks } = await pack(5, 5); + const hex = manifest.chunks[0]!.chunkHash!; + const server = chunkServer(chunks); + const spelled = { ...withChunk(manifest, 0, { chunkHash: `SHA256:${hex.toUpperCase()}` }), contentHash: hex.toUpperCase() }; + assert.deepEqual(await fetchVerifiedContent(ENDPOINT, spelled, { fetch: server.fetch }), body(5)); + assert.equal(new URL(server.calls[0]!.url).searchParams.get("chunkHash"), hex); +}); + +test("a 5.5-chunk manifest returns the input bytes with one credential-free GET per chunk, whatever order the refs are listed in", async () => { + const { manifest, chunks } = await pack(55, 10); + assert.equal(manifest.chunks.length, 6); + for (const refs of [manifest.chunks, [...manifest.chunks].reverse()]) { + const server = chunkServer(chunks); + assert.deepEqual(await fetchVerifiedContent(ENDPOINT, { ...manifest, chunks: refs }, { fetch: server.fetch }), body(55)); + assert.equal(server.calls.length, 6); + assert.deepEqual(new Set(server.calls.map(call => new URL(call.url).searchParams.get("chunkHash"))), new Set(manifest.chunks.map(chunk => chunk.chunkHash))); + for (const call of server.calls) { assert.equal(call.init.method, "GET"); assert.equal(call.init.credentials, "omit"); } + } +}); + +test("with 10 chunks, maxConcurrentChunkRequests 3 peaks at exactly 3 in flight and the default at 4; bad bounds reject RangeError with no fetch", async () => { + const { manifest, chunks } = await pack(100, 10); + for (const [limit, peak] of [[3, 3], [undefined, 4], [1, 1], [32, 10]] as const) { + const server = chunkServer(chunks, 5); + assert.deepEqual(await fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, maxConcurrentChunkRequests: limit }), body(100)); + assert.equal(server.state.peak, peak, `limit ${limit}`); + } + for (const maxConcurrentChunkRequests of [0, 33, 1.5, Number.NaN]) { + const server = chunkServer(chunks); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, maxConcurrentChunkRequests }), RangeError); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, 10, { fetch: server.fetch, maxConcurrentChunkRequests }), RangeError); + assert.equal(server.calls.length, 0); + } + for (const maxBytes of [-1, 1.5, Number.NaN]) await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { maxBytes }), RangeError); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { maxConcurrentChunkRequests: 0 }), /maxConcurrentChunkRequests/); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { maxBytes: -1 }), /maxBytes/); +}); + +test("an empty artifact returns an empty array with no fetch, but still answers to its content hash", async () => { + const { manifest } = await pack(0, 4); + const server = double(() => { throw new Error("no request expected"); }); + const result = await fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch }); + assert.equal(result.length, 0); + await assert.rejects(fetchVerifiedContent(ENDPOINT, { ...manifest, contentHash: "ab".repeat(32) }, { fetch: server.fetch }), codeOf("content-hash-mismatch")); + await assert.rejects(fetchVerifiedContent(ENDPOINT, withChunk(manifest, 0, { chunkHash: "ab".repeat(32) }), { fetch: server.fetch }), codeOf("chunk-hash-mismatch", { chunkHash: "ab".repeat(32) })); + assert.equal(server.calls.length, 0); +}); + +test("a zero-size chunk beside others is manifest-invalid with no fetch", async () => { + const { manifest } = await pack(8, 4); + const [first, second] = manifest.chunks as [typeof manifest.chunks[number], typeof manifest.chunks[number]]; + const flood = { ...manifest, chunks: [first, { ...first, chunkHash: EMPTY_SHA256, offset: 4, sizeBytes: 0 }, second] }; + const server = double(() => { throw new Error("no request expected"); }); + await assert.rejects(fetchVerifiedContent(ENDPOINT, flood, { fetch: server.fetch }), causeIs(/^A zero-size chunk must be the only chunk\.$/)); + // two refs are enough, and the empty one may come last or first + const one = (await pack(4, 4)).manifest; + const empty = { ...one.chunks[0]!, chunkHash: EMPTY_SHA256, sizeBytes: 0 }; + for (const chunks of [[one.chunks[0]!, { ...empty, offset: 4 }], [{ ...empty, offset: 0 }, one.chunks[0]!], [{ ...empty, offset: 0 }, { ...empty, offset: 0 }]]) { + await assert.rejects(fetchVerifiedContent(ENDPOINT, { ...one, sizeBytes: chunks.reduce((sum, chunk) => sum + chunk.sizeBytes, 0), chunks }, { fetch: server.fetch }), codeOf("manifest-invalid")); + } + assert.equal(server.calls.length, 0); +}); + +test("the byte cap is checked before any request: one over maxBytes and one over the 256 MiB default are too-large, the cap itself is not", async () => { + const { manifest, chunks } = await pack(10, 5); + const server = chunkServer(chunks); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, maxBytes: 9 }), codeOf("too-large")); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, 5, { fetch: server.fetch, maxBytes: 4 }), codeOf("too-large")); + assert.equal(server.calls.length, 0); + assert.deepEqual(await fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, maxBytes: 10 }), body(10)); + assert.deepEqual(await fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, 5, { fetch: server.fetch, maxBytes: 5 }), body(5)); + const huge = (size: number) => ({ ...manifest, sizeBytes: size, chunks: [{ ...manifest.chunks[0]!, offset: 0, sizeBytes: size }] }); + const refuses = double(() => { throw new TypeError("offline"); }); + await assert.rejects(fetchVerifiedContent(ENDPOINT, huge(268435457), { fetch: refuses.fetch }), codeOf("too-large")); + assert.equal(refuses.calls.length, 0); + await assert.rejects(fetchVerifiedContent(ENDPOINT, huge(268435456), { fetch: refuses.fetch }), codeOf("network")); +}); + +test("https is required, http only on loopback; ftp and unparseable endpoints are insecure-endpoint with no fetch", async () => { + const { manifest, chunks } = await pack(5, 5); + for (const endpoint of ["http://example.com/c", "http://cdn.example:8080/c", "http://localhost.example/c", "http://127.0.0.1.example/c", "http://10.0.0.1:80/c", "ftp://127.0.0.1/c", "not a url", "", "//cdn.example/c"]) { + const server = chunkServer(chunks); + await assert.rejects(fetchVerifiedContent(endpoint, manifest, { fetch: server.fetch }), codeOf("insecure-endpoint"), endpoint); + await assert.rejects(fetchVerifiedChunk(endpoint, manifest.chunks[0]!.chunkHash!, 5, { fetch: server.fetch }), codeOf("insecure-endpoint"), endpoint); + assert.equal(server.calls.length, 0); + } + for (const endpoint of ["http://127.0.0.1:8080/c", "http://localhost/c", "http://[::1]:9/c", "https://cdn.example/c"]) { + const server = chunkServer(chunks); + assert.deepEqual(await fetchVerifiedContent(endpoint, manifest, { fetch: server.fetch }), body(5), endpoint); + } +}); + +test("non-hex hashes and a gapped manifest are manifest-invalid with no fetch, in both exports", async () => { + const { manifest, chunks } = await pack(10, 5); + const server = chunkServer(chunks); + const bad = [ + withChunk(manifest, 1, { chunkHash: "g".repeat(64) }), + { ...manifest, contentHash: "abc" }, + withChunk(manifest, 1, { offset: 6 }), + { ...manifest, sizeBytes: 11 }, + { ...manifest, sizeBytes: 10.5 }, + // fractional chunk sizes that still tile the manifest, and an infinite one: contiguous, so only a size check refuses them + { ...manifest, chunks: [{ ...manifest.chunks[0]!, sizeBytes: 4 }, { ...manifest.chunks[1]!, offset: 4, sizeBytes: 1.5 }, { ...manifest.chunks[1]!, offset: 5.5, sizeBytes: 4.5 }] }, + { ...manifest, sizeBytes: Infinity, chunks: [{ ...manifest.chunks[0]!, sizeBytes: Infinity }] }, + ]; + for (const candidate of bad) await assert.rejects(fetchVerifiedContent(ENDPOINT, candidate, { fetch: server.fetch }), codeOf("manifest-invalid")); + // the kept cause names which field was wrong + await assert.rejects(fetchVerifiedContent(ENDPOINT, bad[0]!, { fetch: server.fetch }), causeIs(/'chunkHash'/)); + await assert.rejects(fetchVerifiedContent(ENDPOINT, bad[1]!, { fetch: server.fetch }), causeIs(/'contentHash'/)); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, "xyz", 5, { fetch: server.fetch }), causeIs(/'chunkHash'/)); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, 1.5, { fetch: server.fetch }), causeIs(/^Size must be a safe integer\.$/)); + for (const [hash, size] of [["xyz", 5], [manifest.chunks[0]!.chunkHash!, -1], [manifest.chunks[0]!.chunkHash!, 1.5], [manifest.chunks[0]!.chunkHash!, Number.NaN]] as const) { + await assert.rejects(fetchVerifiedChunk(ENDPOINT, hash, size, { fetch: server.fetch }), codeOf("manifest-invalid")); + } + assert.equal(server.calls.length, 0); +}); + +test("every status but 200 (404, 500, 206, 304 and each other 2xx and 3xx tried) is http-status carrying the status, and the body is cancelled", async () => { + const { manifest } = await pack(5, 5); + for (const status of [404, 500, 206, 304, 201, 202, 203, 204, 205, 299, 301]) { + let cancelled = false; + const bodyless = [204, 205, 304].includes(status); + const server = double(() => new Response(bodyless ? null : streamOf(body(5), () => { cancelled = true; }), { status })); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch }), codeOf("http-status", { status, chunkHash: manifest.chunks[0]!.chunkHash! })); + assert.equal(cancelled, !bodyless, String(status)); + } +}); + +test("wrong bytes of the right length are chunk-hash-mismatch naming the chunk; the other in-flight requests are aborted and nothing is returned", async () => { + const { manifest, chunks } = await pack(30, 10); + const signals: AbortSignal[] = []; + const server = double((call, index) => { + signals.push(call.init.signal!); + return index === 0 ? new Response(new Uint8Array(10).fill(9)) : hangUntilAborted(call); + }); + await assert.rejects(within(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, maxConcurrentChunkRequests: 3 })), + codeOf("chunk-hash-mismatch", { chunkHash: chunks[0]!.chunkHash! })); + assert.equal(signals.length, 3); + assert.deepEqual(signals.map(signal => signal.aborted), [true, true, true]); +}); + +test("chunks that each verify under a wrong contentHash are content-hash-mismatch", async () => { + const { manifest, chunks } = await pack(20, 10); + const server = chunkServer(chunks); + await assert.rejects(fetchVerifiedContent(ENDPOINT, { ...manifest, contentHash: await sha256Hex(body(21)) }, { fetch: server.fetch }), codeOf("content-hash-mismatch")); +}); + +test("an endless body is cancelled at the first byte past the chunk, a short body is refused at its end", async () => { + const { manifest } = await pack(10, 10); + let pulled = 0; + let cancelled = false; + const endless = double(() => new Response(new ReadableStream({ + pull(controller) { pulled++; controller.enqueue(Uint8Array.of(1)); }, cancel() { cancelled = true; }, + }, { highWaterMark: 0 }))); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: endless.fetch }), codeOf("length-mismatch")); + assert.ok(pulled <= 11, `pulled ${pulled}`); + assert.ok(cancelled); + for (const length of [0, 9]) { + const short = double(() => new Response(body(10).subarray(0, length) as BodyInit)); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: short.fetch }), codeOf("length-mismatch")); + } + const bodyless = double(() => new Response(null)); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: bodyless.fetch }), codeOf("length-mismatch")); +}); + +test("a fetch TypeError and a mid-body stream error are network, with the cause kept", async () => { + const { manifest } = await pack(10, 10); + const down = new TypeError("fetch failed"); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: double(() => { throw down; }).fetch }), (error: unknown) => codeOf("network")(error) && (error as Error).cause === down); + const boom = new Error("boom"); + const broken = double(() => new Response(new ReadableStream({ start(controller) { controller.enqueue(body(4)); controller.error(boom); } }))); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: broken.fetch }), (error: unknown) => codeOf("network")(error) && (error as Error).cause === boom); +}); + +test("a pre-aborted signal rejects with its reason and no fetch; an abort mid-flight rejects with the same reason and starts no further fetch", async () => { + const { manifest } = await pack(100, 10); + const reason = new Error("caller gave up"); + const server = double(hangUntilAborted); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, signal: AbortSignal.abort(reason) }), (error: unknown) => error === reason); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, 10, { fetch: server.fetch, signal: AbortSignal.abort(reason) }), (error: unknown) => error === reason); + assert.equal(server.calls.length, 0); + const controller = new AbortController(); + const pending = fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, signal: controller.signal, maxConcurrentChunkRequests: 2 }); + for (let waited = 0; server.calls.length < 2 && waited < 2000; waited++) await new Promise(resolve => setTimeout(resolve, 1)); + assert.equal(server.calls.length, 2, "two requests are in flight before the abort"); + controller.abort(reason); + await assert.rejects(within(pending), (error: unknown) => error === reason); + await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(server.calls.length, 2); +}); + +test("with crypto.subtle shadowed, both exports are webcrypto-unavailable before any fetch", async () => { + const { manifest, chunks } = await pack(5, 5); + const server = chunkServer(chunks); + const original = Object.getOwnPropertyDescriptor(globalThis, "crypto")!; + try { + for (const value of [{}, undefined]) { + Object.defineProperty(globalThis, "crypto", { value, configurable: true }); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch }), codeOf("webcrypto-unavailable")); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, 5, { fetch: server.fetch }), codeOf("webcrypto-unavailable")); + } + } finally { Object.defineProperty(globalThis, "crypto", original); } + assert.equal(server.calls.length, 0); +}); + +test("fetchVerifiedChunk sends the packed record key, verifies, sends nothing for size 0, and refuses wrong bytes", async () => { + const { manifest, chunks } = await pack(10, 10); + const hex = manifest.chunks[0]!.chunkHash!; + const server = chunkServer(chunks); + assert.deepEqual(await fetchVerifiedChunk(ENDPOINT, hex.toUpperCase(), 10, { fetch: server.fetch }), body(10)); + assert.equal(server.calls[0]!.url, `${ENDPOINT}?chunkHash=${hex}&recordKey=mesh%3Acdn%3Achunk%3A${hex}`); + assert.equal(server.calls[0]!.init.credentials, "omit"); + const none = double(() => { throw new Error("no request expected"); }); + assert.equal((await fetchVerifiedChunk(ENDPOINT, EMPTY_SHA256, 0, { fetch: none.fetch })).length, 0); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, hex, 0, { fetch: none.fetch }), codeOf("chunk-hash-mismatch", { chunkHash: hex })); + assert.equal(none.calls.length, 0); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, hex, 10, { fetch: double(() => new Response(new Uint8Array(10))).fetch }), codeOf("chunk-hash-mismatch", { chunkHash: hex })); +}); + +test("no error message contains the endpoint, its secret query, or a body byte", async () => { + const secret = "https://cdn.example/c?token=CANARYSECRET"; + const { manifest } = await pack(10, 10); + const canary = new TextEncoder().encode("BODYCANARY"); + const doubles: [CultMeshContentErrorCode, Handler | undefined][] = [ + ["http-status", () => new Response(canary, { status: 500 })], + ["chunk-hash-mismatch", () => new Response(canary)], + ["length-mismatch", () => new Response(new Uint8Array(40).fill(66))], + ["network", () => { throw new TypeError(`failed ${secret}`); }], + ["insecure-endpoint", undefined], + ]; + for (const [code, handler] of doubles) { + const server = double(handler ?? (() => new Response())); + const error = await fetchVerifiedContent(code === "insecure-endpoint" ? "http://example.com/c?token=CANARYSECRET" : secret, manifest, { fetch: server.fetch }).then(() => undefined, reason => reason); + assert.ok(codeOf(code)(error), code); + for (const text of [(error as Error).message, String(error)]) { assert.doesNotMatch(text, /CANARY|cdn\.example|token/); } + } +}); + +test("every error code has its own fixed message, and a cause only when one was given", () => { + const codes: CultMeshContentErrorCode[] = ["manifest-invalid", "too-large", "insecure-endpoint", "webcrypto-unavailable", "network", "http-status", "length-mismatch", "chunk-hash-mismatch", "content-hash-mismatch"]; + const messages = codes.map(code => new CultMeshContentError(code).message); + assert.equal(new Set(messages).size, codes.length); + assert.ok(messages.every(message => message.length > 0 && !/https?:/.test(message))); + assert.equal("cause" in new CultMeshContentError("network"), false); + assert.equal(new CultMeshContentError("network").name, "CultMeshContentError"); + assert.equal(new CultMeshContentError("network", undefined, undefined, 0).cause, 0); +}); + +test("the caller's abort listener is added once and removed again, after success and after failure", async () => { + const { manifest, chunks } = await pack(10, 5); + for (const handler of [undefined, () => new Response(null, { status: 500 })]) { + const controller = new AbortController(); + const added: unknown[] = []; + const removed: unknown[] = []; + const add = controller.signal.addEventListener.bind(controller.signal); + const remove = controller.signal.removeEventListener.bind(controller.signal); + controller.signal.addEventListener = ((type: string, listener: EventListener, options?: unknown) => { added.push([type, listener]); add(type, listener, options as AddEventListenerOptions); }) as typeof add; + controller.signal.removeEventListener = ((type: string, listener: EventListener) => { removed.push([type, listener]); remove(type, listener); }) as typeof remove; + const server = handler ? double(handler) : chunkServer(chunks); + await fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch, signal: controller.signal }).catch(() => undefined); + assert.equal(added.length, 1); + assert.deepEqual(removed, added); + assert.equal((added[0] as [string])[0], "abort"); + } +}); + +interface Stub { endpoint: string; close(): Promise } +async function stub(handler: (request: IncomingMessage, response: ServerResponse) => void): Promise { + const server = createServer(handler); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + return { + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}/content`, + close: () => new Promise(resolve => { server.close(() => resolve()); server.closeAllConnections(); }), + }; +} + +test("stub server through the real global fetch: gzip verifies, a short body is network, an endless body is cut, 404 is http-status", async () => { + const { manifest, chunks } = await pack(1000, 1000); + const payload = chunks[0]!.payload; + const modes = new Map void>(); + const server = await stub((request, response) => modes.get(new URL(request.url!, "http://x").searchParams.get("tenant")!)!(request, response)); + const via = (tenant: string) => fetchVerifiedContent(`${server.endpoint}?tenant=${tenant}`, manifest); + try { + modes.set("gzip", (_, response) => { + const encoded = gzipSync(payload); + response.writeHead(200, { "content-encoding": "gzip", "content-length": encoded.length, "access-control-allow-origin": "*" }); + response.end(encoded); + }); + assert.deepEqual(await via("gzip"), payload); + + modes.set("short", (_, response) => { + response.writeHead(200, { "content-length": 1000 }); + response.write(payload.subarray(0, 400)); + setTimeout(() => response.socket!.destroy(), 20); + }); + await assert.rejects(via("short"), codeOf("network")); + + let sawClose!: () => void; + const closed = new Promise(resolve => { sawClose = resolve; }); + modes.set("endless", (_, response) => { + response.writeHead(200); + const timer = setInterval(() => response.write(new Uint8Array(1000)), 1); + response.on("close", () => { clearInterval(timer); sawClose(); }); + }); + await assert.rejects(via("endless"), codeOf("length-mismatch")); + await closed; + + modes.set("missing", (_, response) => { response.writeHead(404, { "access-control-allow-origin": "*" }); response.end("nope"); }); + await assert.rejects(via("missing"), codeOf("http-status", { status: 404 })); + } finally { await server.close(); } +}); + +const flipped = (bytes: Uint8Array, at: number): Uint8Array => { const copy = bytes.slice(); copy[at] ^= 1; return copy; }; + +test("wrong bytes of the right length are refused at every size, whichever byte differs: both exports, one chunk and several", async () => { + for (const size of [1, 999, 1000, 1001, 65537, 4 * 1024 * 1024 + 1]) { + const { manifest, chunks } = await pack(size, size); + const hash = manifest.chunks[0]!.chunkHash!; + for (const at of [0, size >> 1, size - 1]) { + const wrong = double(() => new Response(flipped(chunks[0]!.payload, at) as BodyInit)); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, hash, size, { fetch: wrong.fetch }), codeOf("chunk-hash-mismatch", { chunkHash: hash }), `chunk ${size} byte ${at}`); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: wrong.fetch }), codeOf("chunk-hash-mismatch", { chunkHash: hash }), `content ${size} byte ${at}`); + } + } + const { manifest, chunks } = await pack(4500, 1500); + const middle = manifest.chunks[1]!.chunkHash!; + const wrongMiddle = double(call => { + const asked = new URL(call.url).searchParams.get("chunkHash"); + const payload = chunks.find(chunk => chunk.chunkHash === asked)!.payload; + return new Response((asked === middle ? flipped(payload, 1499) : payload) as BodyInit); + }); + await assert.rejects(fetchVerifiedContent(ENDPOINT, manifest, { fetch: wrongMiddle.fetch }), codeOf("chunk-hash-mismatch", { chunkHash: middle })); +}); + +test("a body of the wrong length in one chunk of several is length-mismatch naming that chunk, whichever way it is wrong", async () => { + const { manifest, chunks } = await pack(30, 10); + const target = manifest.chunks[2]!.chunkHash!; + for (const length of [0, 9, 11, 20]) { + const server = double(call => { + const asked = new URL(call.url).searchParams.get("chunkHash"); + const payload = chunks.find(chunk => chunk.chunkHash === asked)!.payload; + return new Response((asked === target ? new Uint8Array(length).fill(payload[0]!) : payload) as BodyInit); + }); + await assert.rejects(within(fetchVerifiedContent(ENDPOINT, manifest, { fetch: server.fetch })), codeOf("length-mismatch", { chunkHash: target }), `length ${length}`); + } +}); + +test("a failure is thrown without waiting for a fetch whose body ignores the abort, and so is a caller abort", async () => { + const { manifest } = await pack(30, 10); + const stalled = () => new Response(new ReadableStream({ start(controller) { controller.enqueue(body(3)); } })); + const firstWrong = double((_, index) => index === 0 ? new Response(new Uint8Array(10)) : stalled()); + await assert.rejects(within(fetchVerifiedContent(ENDPOINT, manifest, { fetch: firstWrong.fetch, maxConcurrentChunkRequests: 3 })), codeOf("chunk-hash-mismatch")); + const reason = new Error("stop waiting"); + const never = double(() => new Promise(() => {})); + const early = new AbortController(); + const noResponse = fetchVerifiedContent(ENDPOINT, manifest, { fetch: never.fetch, signal: early.signal }); + early.abort(reason); + await assert.rejects(within(noResponse), (error: unknown) => error === reason); + const stalledBodies = double(stalled); + const late = new AbortController(); + const waiting = fetchVerifiedContent(ENDPOINT, manifest, { fetch: stalledBodies.fetch, signal: late.signal }); + for (let waited = 0; stalledBodies.calls.length < 3 && waited < 2000; waited++) await new Promise(resolve => setTimeout(resolve, 1)); + late.abort(reason); + await assert.rejects(within(waiting), (error: unknown) => error === reason); +}); + +test("an abort that lands while the last bytes are hashed rejects with its reason: no success for a cancelled caller", async () => { + const { manifest, chunks } = await pack(10, 10); + const subtle = globalThis.crypto.subtle; + const digest = subtle.digest.bind(subtle); + const reason = new Error("cancelled late"); + const hash = manifest.chunks[0]!.chunkHash!; + try { + // digest 1 hashes the only chunk, digest 2 the whole body; fetchVerifiedChunk has no second digest + for (const [name, abortOnDigest, run] of [ + ["content", 1, (signal: AbortSignal) => fetchVerifiedContent(ENDPOINT, manifest, { fetch: chunkServer(chunks).fetch, signal })], + ["content", 2, (signal: AbortSignal) => fetchVerifiedContent(ENDPOINT, manifest, { fetch: chunkServer(chunks).fetch, signal })], + ["chunk", 1, (signal: AbortSignal) => fetchVerifiedChunk(ENDPOINT, hash, 10, { fetch: chunkServer(chunks).fetch, signal })], + ] as const) { + const controller = new AbortController(); + let digests = 0; + Object.defineProperty(subtle, "digest", { configurable: true, writable: true, value: (...args: Parameters) => { if (++digests === abortOnDigest) controller.abort(reason); return digest(...args); } }); + const outcome = await run(controller.signal).then(() => "resolved", error => error); + assert.equal(outcome, reason, `${name} abort on digest ${abortOnDigest}`); + } + } finally { delete (subtle as unknown as Record).digest; } +}); + +test("a size the runtime cannot allocate is too-large in both exports, not a RangeError, and nothing is requested", async () => { + const { manifest } = await pack(10, 10); + const size = Number.MAX_SAFE_INTEGER; + const huge = { ...manifest, sizeBytes: size, chunks: [{ ...manifest.chunks[0]!, offset: 0, sizeBytes: size }] }; + const server = double(() => { throw new Error("no request expected"); }); + await assert.rejects(fetchVerifiedContent(ENDPOINT, huge, { fetch: server.fetch, maxBytes: size }), codeOf("too-large")); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, size, { fetch: server.fetch, maxBytes: size }), codeOf("too-large")); + assert.equal(server.calls.length, 0); +}); + +test("redirects are refused: the fetch is told so, and a real redirect to another origin is network with nothing requested there", async () => { + const { manifest, chunks } = await pack(10, 10); + const seen = chunkServer(chunks); + await fetchVerifiedContent(ENDPOINT, manifest, { fetch: seen.fetch }); + assert.equal(seen.calls[0]!.init.redirect, "error"); + let elsewhereCalls = 0; + const elsewhere = await stub((_, response) => { elsewhereCalls++; response.end(chunks[0]!.payload); }); + const here = await stub((_, response) => { response.writeHead(302, { location: `${elsewhere.endpoint}?x=1` }); response.end(); }); + try { + await assert.rejects(fetchVerifiedContent(here.endpoint, manifest), codeOf("network")); + await assert.rejects(fetchVerifiedChunk(here.endpoint, manifest.chunks[0]!.chunkHash!, 10), codeOf("network")); + assert.equal(elsewhereCalls, 0); + } finally { await here.close(); await elsewhere.close(); } +}); + +test("esbuild bundles dist/content.js with platform 'browser', and a node:crypto import fails the same bundle", async () => { + const entry = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "dist", "content.js"); + const options = { bundle: true, platform: "browser", format: "esm", write: false, logLevel: "silent" } as const; + const result = await build({ ...options, entryPoints: [entry] }); + assert.ok(result.outputFiles[0]!.text.includes("CultMeshContentError")); + await assert.rejects(build({ ...options, stdin: { contents: `import "node:crypto"; import ${JSON.stringify(entry)};`, resolveDir: dirname(entry) } })); +}); diff --git a/packages/cultnet-ts/src/content.ts b/packages/cultnet-ts/src/content.ts index 323a59e9..754ac3f2 100644 --- a/packages/cultnet-ts/src/content.ts +++ b/packages/cultnet-ts/src/content.ts @@ -100,13 +100,18 @@ export async function artifactRecordKey(artifactId: string | null, version: stri // ---- validation ------------------------------------------------------------------------------------------------------ -/** `ValidateManifestShape`, in its order and with its messages, then the metadata keys a dictionary could not hold twice. */ +/** + * `ValidateManifestShape`, in its order and with its messages, then the metadata keys a dictionary could not hold twice. + * C# sizes are `long`, so a fraction, NaN or Infinity cannot exist on the wire; a hand-built manifest is refused here, once. + */ export function validateManifest(manifest: CultMeshCdnArtifactManifest): void { if (manifest.sizeBytes < 0) throw new Error("CDN artifact manifest has a negative size."); + if (!Number.isInteger(manifest.sizeBytes)) throw new Error("CDN artifact manifest size is not a whole number."); let expectedOffset = 0; for (const reference of [...manifest.chunks].sort((a, b) => a.offset - b.offset)) { if (reference.offset !== expectedOffset) throw new Error("CDN artifact chunks are not contiguous."); if (reference.sizeBytes < 0) throw new Error("CDN artifact chunk has a negative size."); + if (!Number.isInteger(reference.sizeBytes)) throw new Error("CDN artifact chunk size is not a whole number."); expectedOffset += reference.sizeBytes; } if (expectedOffset !== manifest.sizeBytes) throw new Error("CDN artifact chunk sizes do not sum to the manifest size."); diff --git a/packages/cultnet-ts/test/content.test.ts b/packages/cultnet-ts/test/content.test.ts index 807764fa..6d30b7d2 100644 --- a/packages/cultnet-ts/test/content.test.ts +++ b/packages/cultnet-ts/test/content.test.ts @@ -231,6 +231,23 @@ test("validateManifest refuses negative size, a gap, an overlap and a size-sum m validateManifest(manifestOf({ sizeBytes: 0, chunks: [] })); }); +test("validateManifest refuses a size that is not a whole number: a fraction, NaN or Infinity, in the manifest or in a chunk", () => { + const chunk = (offset: number, sizeBytes: number) => ({ chunkHash: HASH, offset, sizeBytes, recordKey: "k" }); + const refuses = (over: Partial, message: string) => + assert.throws(() => validateManifest(manifestOf(over)), new Error(message)); + const manifestMessage = "CDN artifact manifest size is not a whole number."; + const chunkMessage = "CDN artifact chunk size is not a whole number."; + for (const size of [1.5, Number.NaN, Number.POSITIVE_INFINITY]) refuses({ sizeBytes: size, chunks: [] }, manifestMessage); + // fractional chunk sizes that still tile the manifest are contiguous, so only the whole-number rule refuses them + refuses({ sizeBytes: 6, chunks: [chunk(0, 4), chunk(4, 1.5), chunk(5.5, 0.5)] }, chunkMessage); + refuses({ sizeBytes: 1.5, chunks: [chunk(0, 1.5)] }, manifestMessage); + refuses({ sizeBytes: 4, chunks: [chunk(0, Number.NaN)] }, chunkMessage); + refuses({ sizeBytes: Number.POSITIVE_INFINITY, chunks: [chunk(0, Number.POSITIVE_INFINITY)] }, manifestMessage); + refuses({ sizeBytes: 4, chunks: [chunk(0, 2), chunk(2, Number.POSITIVE_INFINITY)] }, chunkMessage); + refuses({ sizeBytes: 0, chunks: [chunk(0, -1)] }, "CDN artifact chunk has a negative size."); + validateManifest(manifestOf({ sizeBytes: 2 ** 53 - 1, chunks: [chunk(0, 2 ** 53 - 1)] })); +}); + test("normalizeHash: 'SHA256:ABC ' -> 'abc', ' sha256:abc' -> 'sha256:abc', blank throws", () => { assert.equal(normalizeHash("SHA256:ABC "), "abc"); assert.equal(normalizeHash("sHa256:AbC"), "abc");