diff --git a/packages/cultnet-ts/src/content.ts b/packages/cultnet-ts/src/content.ts index 7e9ba6ba..8474c396 100644 --- a/packages/cultnet-ts/src/content.ts +++ b/packages/cultnet-ts/src/content.ts @@ -10,6 +10,7 @@ import { encode } from "@msgpack/msgpack"; import { checkPayloadDepth } from "@gamecult/cultcache-ts/document"; import { isNullOrWhiteSpaceCSharp, trimCSharp } from "./cultmesh-authority"; +import { wellFormed } from "./utf8-bound"; export const CULTMESH_CDN_MANIFEST_SCHEMA = "gamecult.mesh.cdn_artifact_manifest.v1"; export const DEFAULT_CHUNK_SIZE_BYTES = 4 * 1024 * 1024; @@ -125,15 +126,18 @@ const mapHeader = (count: number): Uint8Array => : count < 0x10000 ? Uint8Array.of(0xde, count >> 8, count & 0xff) : Uint8Array.of(0xdf, count >>> 24, (count >> 16) & 0xff, (count >> 8) & 0xff, count & 0xff); +/** A string slot is written as C# writes it: a lone surrogate has no UTF-8 encoding and becomes U+FFFD at every length. */ +const text = (field: T): T => (typeof field === "string" ? wellFormed(field) : field) as T; + /** The positional ten-array of the C# `[MessagePackObject]`: chunk references are four-arrays, metadata a map in entry order. */ export function encodeManifest(manifest: CultMeshCdnArtifactManifest): Uint8Array { const { artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc, chunks, tags, metadata } = manifest; return concat([ Uint8Array.of(0x9a), - ...[artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc].map(field => encode(field)), - encode(chunks.map(c => [c.chunkHash, c.offset, c.sizeBytes, c.recordKey])), - encode(tags), - ...(metadata === null ? [encode(null)] : [mapHeader(metadata.length), ...metadata.flatMap(([key, value]) => [encode(key), encode(value)])]), + ...[artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc].map(field => encode(text(field))), + encode(chunks.map(c => [text(c.chunkHash), c.offset, c.sizeBytes, text(c.recordKey)])), + encode(tags === null ? null : tags.map(text)), + ...(metadata === null ? [encode(null)] : [mapHeader(metadata.length), ...metadata.flatMap(([key, value]) => [encode(text(key)), encode(text(value))])]), ]); } diff --git a/packages/cultnet-ts/src/utf8-bound.ts b/packages/cultnet-ts/src/utf8-bound.ts index d21ee849..008c1657 100644 --- a/packages/cultnet-ts/src/utf8-bound.ts +++ b/packages/cultnet-ts/src/utf8-bound.ts @@ -8,13 +8,18 @@ * A value is therefore in bounds exactly when this returns it unchanged. */ export function truncateUtf8Bytes(value: string, maxBytes: number): string { - const wellFormed = (value as string & { toWellFormed(): string }).toWellFormed(); + const text = wellFormed(value); let bytes = 0; let end = 0; - for (const char of wellFormed) { + for (const char of text) { bytes += Buffer.byteLength(char, "utf8"); if (bytes > maxBytes) break; end += char.length; } - return wellFormed.slice(0, end); + return text.slice(0, end); +} + +/** `value` with every lone surrogate replaced by U+FFFD: the one spelling every TypeScript writer puts on a wire. */ +export function wellFormed(value: string): string { + return (value as string & { toWellFormed(): string }).toWellFormed(); } diff --git a/packages/cultnet-ts/test/content.test.ts b/packages/cultnet-ts/test/content.test.ts index 490eec9d..807764fa 100644 --- a/packages/cultnet-ts/test/content.test.ts +++ b/packages/cultnet-ts/test/content.test.ts @@ -389,3 +389,28 @@ test("a refusal names its true cause", () => { refuses(Uint8Array.of(0x91, 0xa3, 0x41), /ends before its last field/); refuses(Uint8Array.from([0x9b, ...wire().slice(1), 0xc1]), /field 'manifest' has the wrong type/); }); + +test("a lone surrogate is written as U+FFFD at every string length, short and long alike, in every string slot", async () => { + const base = (await packContent("x", Uint8Array.of(1, 2, 3), { createdAtUtc: "2026-09-22T00:00:00.0000000+00:00" })).manifest; + // @msgpack/msgpack switches encoders past 50 UTF-16 units; the reference writes U+FFFD on both sides of that line. + for (const length of [1, 2, 16, 31, 32, 49, 50, 51, 52, 64, 200]) { + for (const [lone, replaced] of [["\ud800", "\ufffd"], ["\udfff", "\ufffd"]]) { + const filler = "a".repeat(length - 1); + const bad = `${filler}${lone}`; + const good = `${filler}${replaced}`; + const withText = (value: string): CultMeshCdnArtifactManifest => ({ + ...base, + artifactId: value, kind: value, version: value, mimeType: value, createdAtUtc: value, + chunks: [{ ...base.chunks[0]!, recordKey: value }], + tags: [value], + metadata: [[value, value]], + }); + const bytes = encodeManifest(withText(bad)); + assert.deepEqual(bytes, encodeManifest(withText(good)), `length ${length}`); + assert.ok(!bytes.some((b, i) => b === 0xed && bytes[i + 1]! >= 0xa0 && bytes[i + 1]! <= 0xbf), `no CESU surrogate bytes at length ${length}`); + assert.equal(decodeManifest(bytes).artifactId, good, `length ${length} reads back as written`); + } + } + const packed = await packContent(`a${"b".repeat(60)}\ud800`, Uint8Array.of(1), { kind: "k\udc00", createdAtUtc: "2026-09-22T00:00:00.0000000+00:00" }); + assert.equal(decodeManifest(encodeManifest(packed.manifest)).kind, "k\ufffd"); +});