diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index 520a4836..c64da8de 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -103,6 +103,15 @@ jobs: # Linux at all, and the browser client's never ran in CI on any OS. # packages/cultnet-ts verifies both vector files; cultmesh-browser proves # the browser client reads the shared rules rather than its own. + # No workflow ran cultcache-ts's tests, and its test script listed two files by hand while a third sat compiled + # and unrun. The wiring check fails when any TypeScript test file is run by no script, and the cache's own suite + # runs here beside the packages built on it. + - name: Check that every TypeScript test file is run by its package + run: node scripts/check-typescript-tests-wired.mjs + + - name: Verify the CultCache document owner with the TypeScript cache + run: npm run test --workspace packages/cultcache-ts + - name: Verify the CultMesh authority rules with the TypeScript verifier run: npm run test --workspace packages/cultnet-ts diff --git a/package-lock.json b/package-lock.json index d07fbc78..403865c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1202,6 +1202,7 @@ "version": "0.3.0", "license": "MIT", "dependencies": { + "@gamecult/cultcache-ts": "^0.14.0", "cultnet-ts": "^0.2.0" }, "devDependencies": { diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 1ea366f0..d37fa047 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -32,6 +32,28 @@ This is not an ORM in a fake mustache. It is for cases where you want: That keeps the polymorphic boundary tight without making callers write boilerplate by candlelight. +## Reading And Writing Payloads + +A document's bytes travel as a `DocumentPayload`: in a store envelope, a CultNet record and a +realtime frame. At run time it is the same `Uint8Array`, so nothing on the wire changes. At compile +time it is opaque: it is assignable to and from neither `Uint8Array` nor `ArrayLike`, so no +MessagePack decoder accepts it. + +- `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter, + codec or plain map and then its schema. The definition is required. +- `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter, + codec or plain map. +- `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a + writer of pre-encoded or signed bytes) and `documentPayloadBytes(payload)` is where they leave (a + store write, a frame encode, hashing or signing). Reach for them only at those edges. + +```ts +const note = decodeDocumentPayload(noteDocument, envelope.payload); // parsed Note +const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload +``` + +Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. + ## `.cc` Inspection CultCache files may use the `.cc` extension. The bytes are still the canonical diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index d31ab1b8..852c047e 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -10,12 +10,24 @@ "require": "./dist/index.js", "default": "./dist/index.js" }, + "./document": { + "types": "./dist/document.d.ts", + "require": "./dist/document.js", + "default": "./dist/document.js" + }, "./inspection": { "types": "./dist/cult-cache-inspector.d.ts", "require": "./dist/cult-cache-inspector.js", "default": "./dist/cult-cache-inspector.js" } }, + "typesVersions": { + "*": { + "document": [ + "dist/document.d.ts" + ] + } + }, "files": [ "dist" ], @@ -23,7 +35,7 @@ "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "prepack": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --pretty false", - "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/cult-cache.test.js dist-test/test/union-parity.test.js" + "test": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json && node ../../node_modules/typescript/bin/tsc -p tsconfig.test.json && node --test dist-test/test/*.test.js" }, "keywords": [ "messagepack", diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 94247d49..c0ee0b58 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,10 +1,17 @@ -import { resolveDocumentFormatter, resolveDocumentMembers } from "./document"; +import { + decodeDocumentPayload, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, + resolveDocumentFormatter, + resolveDocumentMembers, +} from "./document"; +import type { DocumentPayload } from "./document"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheDocumentAccessor, CultCacheDocumentDefinition, - CultCacheDocumentFormatter, CultCacheDocumentRegistry, CultCacheDocumentValue, CultCacheEnvelope, @@ -14,10 +21,16 @@ import type { type Accessor = (value: unknown) => string | undefined; +// A registered definition reads and writes its documents through the owner, on payloads the cache has cloned. +type RegisteredFormatter = { + encode(value: unknown): DocumentPayload; + decode(payload: DocumentPayload): unknown; +}; + type RegisteredDefinition = { readonly definition: AnyCultCacheDocumentDefinition; readonly global: boolean; - readonly formatter: CultCacheDocumentFormatter; + readonly formatter: RegisteredFormatter; readonly catalogEntry: CultCacheSchemaCatalogEntry; nameAccessor?: Accessor; indexAccessors: ReadonlyMap; @@ -674,18 +687,13 @@ export class CultCache { } } - #createFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { - const userFormatter = resolveDocumentFormatter(definition); + #createFormatter(definition: AnyCultCacheDocumentDefinition): RegisteredFormatter { + // A definition that declares both a codec and a formatter is refused here, where it is registered. + resolveDocumentFormatter(definition); return { - encode: (value: unknown) => { - const parsed = definition.schema.parse(value); - return this.#cloneBytes(userFormatter.encode(parsed)); - }, - decode: (payload: Uint8Array) => { - const decoded = userFormatter.decode(this.#cloneBytes(payload)); - return definition.schema.parse(decoded); - }, + encode: (value: unknown) => this.#cloneBytes(encodeDocumentPayload(definition, value)), + decode: (payload: DocumentPayload) => decodeDocumentPayload(definition, this.#cloneBytes(payload)), }; } @@ -1026,7 +1034,7 @@ export class CultCache { return `${type}::${indexName}`; } - #cloneBytes(payload: Uint8Array): Uint8Array { - return new Uint8Array(payload); + #cloneBytes(payload: DocumentPayload): DocumentPayload { + return documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(payload))); } } diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 50e08afb..183bd110 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -78,7 +78,7 @@ export function defineDocumentType( // The one place that decides how a document is encoded: its own formatter, or its codec's, or a plain map. A codec and a // formatter are two answers to one question, so a definition that carries both is refused wherever it enters (the -// builder resolves through here as well). +// builder resolves through here as well). Package-internal: not exported from the package index. export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinition): CultCacheDocumentFormatter { if (definition.codec && definition.formatter) { throw new Error(`CultCache document type "${definition.type}" declares both a codec and a formatter.`); @@ -91,13 +91,44 @@ const plainDocumentFormatter: CultCacheDocumentFormatter = { decode: (payload: Uint8Array) => decode(payload), }; +/** + * The bytes of one document, as they travel in a store envelope, a CultNet record or a realtime frame. + * + * At run time this is the same `Uint8Array`, so frames encode as they always did. At compile time it is opaque: it is not + * assignable to or from `Uint8Array` or `ArrayLike`, so no byte decoder accepts it. The only reader is + * `decodeDocumentPayload(definition, payload)`, which needs the definition that says how the bytes are laid out, and the + * only writer is `encodeDocumentPayload(definition, value)`. The two escapes are named: `documentPayloadFromBytes` where + * bytes enter (a validated record, a store read, tests, a writer of pre-encoded or signed bytes) and + * `documentPayloadBytes` where they leave (a store write, a frame encode, hashing or signing). + * + * The key is a plain string, not a unique symbol, so two installed copies of this package stay compatible. + */ +export interface DocumentPayload { + readonly "cultcache.documentPayload": true; + readonly byteLength: number; +} + +export function documentPayloadFromBytes(bytes: Uint8Array): DocumentPayload { + return bytes as unknown as DocumentPayload; +} + +export function documentPayloadBytes(payload: DocumentPayload): Uint8Array { + return payload as unknown as Uint8Array; +} + // The one reader of a document payload: through the definition's formatter, codec or plain map, then its schema. A // reader holds the definition or it has no business reading the bytes, so there is no definition-less form. export function decodeDocumentPayload( definition: TDefinition, - payload: Uint8Array, + payload: DocumentPayload, ): CultCacheDocumentValue { - return definition.schema.parse(resolveDocumentFormatter(definition).decode(payload)) as CultCacheDocumentValue; + const decoded = resolveDocumentFormatter(definition).decode(documentPayloadBytes(payload)); + return definition.schema.parse(decoded) as CultCacheDocumentValue; +} + +// The one writer: the definition's schema first, then its formatter, codec or plain map. +export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition, value: unknown): DocumentPayload { + return documentPayloadFromBytes(resolveDocumentFormatter(definition).encode(definition.schema.parse(value))); } // The schema members of a document: the ones it declares, else those its codec derives, else none. diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 8fae6f4c..c5dfbff1 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,15 @@ export * as codecs from "./codec"; export type { Codec, SlotsCodec } from "./codec"; export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { decodeDocumentPayload, defineDocumentRegistry, defineDocumentType, resolveDocumentFormatter } from "./document"; +export { + decodeDocumentPayload, + defineDocumentRegistry, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, +} from "./document"; +export type { DocumentPayload } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; export type { AnyCultCacheDocumentDefinition, diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index b7df1454..cdc6cf9e 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -4,6 +4,7 @@ import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; +import { documentPayloadBytes, documentPayloadFromBytes } from "./document"; import { STORE_FORMAT_VERSION, isV1Snapshot, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, @@ -62,7 +63,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { type: catalogEntry.schemaName, schemaId: record.schemaId, catalogEntry, - payload: record.payload, + payload: documentPayloadFromBytes(record.payload), storedAt: record.storedAt, }; }); @@ -82,7 +83,10 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { payload, }; }); - const parsed = envelopeArraySchema.parse(normalized) as CultCacheEnvelope[]; + const parsed: CultCacheEnvelope[] = envelopeArraySchema.parse(normalized).map((entry) => ({ + ...entry, + payload: documentPayloadFromBytes(entry.payload), + })); if (repairedLegacyPayload) { await this.#writeAll(parsed); @@ -232,7 +236,7 @@ function encodeSnapshot(entries: CultCacheEnvelope[]): unknown[] { entry.key, schemaIdForEnvelope(entry), entry.storedAt, - entry.payload, + documentPayloadBytes(entry.payload), ]); return [ diff --git a/packages/cultcache-ts/src/types.ts b/packages/cultcache-ts/src/types.ts index dada7fc0..88c20f2e 100644 --- a/packages/cultcache-ts/src/types.ts +++ b/packages/cultcache-ts/src/types.ts @@ -1,4 +1,5 @@ import type { SlotsCodec } from "./codec"; +import type { DocumentPayload } from "./document"; export type CultCacheIndexScalar = string | number | boolean | bigint; @@ -80,7 +81,7 @@ export interface CultCacheDocumentRegistry< export interface CultCacheEnvelope { key: string; type: string; - payload: Uint8Array; + payload: DocumentPayload; storedAt: string; schemaId?: string; catalogEntry?: CultCacheSchemaCatalogEntry; diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index b7c51cc1..dda2aa79 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -13,7 +13,7 @@ import { z } from "zod"; import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; -import { defineDocumentRegistry, defineDocumentType } from "../src/document"; +import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; @@ -202,7 +202,7 @@ test("CultCache fails closed on unknown or illegal persisted polymorphic state", { key: "mystery", type: "forbidden", - payload: encode({ theme: "ash" }), + payload: documentPayloadFromBytes(encode({ theme: "ash" })), storedAt: new Date().toISOString(), }, ])) @@ -219,13 +219,13 @@ test("CultCache fails closed on unknown or illegal persisted polymorphic state", { key: "one", type: "settings", - payload: encode({ theme: "ash" }), + payload: documentPayloadFromBytes(encode({ theme: "ash" })), storedAt: new Date().toISOString(), }, { key: "two", type: "settings", - payload: encode({ theme: "ember" }), + payload: documentPayloadFromBytes(encode({ theme: "ember" })), storedAt: new Date().toISOString(), }, ])) @@ -650,7 +650,7 @@ async function storeState(path: string): Promise { return []; } return (await new SingleFileMessagePackBackingStore(path).pullAll()) - .map((entry) => `${entry.type}:${entry.key}:${Buffer.from(entry.payload).toString("hex")}`) + .map((entry) => `${entry.type}:${entry.key}:${Buffer.from(documentPayloadBytes(entry.payload)).toString("hex")}`) .sort(); } @@ -821,8 +821,8 @@ test("concurrentGlobalPutsCannotBothReachDisk", async () => { const onDisk = await inner.pullAll(); assert.deepEqual(onDisk.map((entry) => entry.key), [CultCache.GLOBAL_KEY]); assert.deepEqual( - Buffer.from(onDisk[0].payload), - Buffer.from(cache.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY).payload), + Buffer.from(documentPayloadBytes(onDisk[0].payload)), + Buffer.from(documentPayloadBytes(cache.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY).payload)), ); assert.deepEqual(cache.getRequiredGlobal(settingsDocument), { t: "second" }); const reloaded = build(new SingleFileMessagePackBackingStore(storePath)); @@ -1042,7 +1042,7 @@ test("pullWithThrowingIndexAccessorLeavesCacheUnchanged", async () => { await new SingleFileMessagePackBackingStore(storePath).push({ ...potion, key: "bomb", - payload: encode({ name: "Bomb", category: "explosive" }), + payload: documentPayloadFromBytes(encode({ name: "Bomb", category: "explosive" })), }); await assert.rejects(cache.pullAllBackingStores(), /category accessor refuses Bomb/u); @@ -1349,7 +1349,7 @@ async function writeTsInteropStore( members: interopNoteDocument.members, }, storedAt: new Date().toISOString(), - payload, + payload: documentPayloadFromBytes(payload), }); return note; } @@ -1524,7 +1524,7 @@ test("SingleFileMessagePackBackingStore refuses a dangling symlink at its path a const file = join(dir, "store.cc"); await symlink(join(volume, "store.cc"), file); const store = new SingleFileMessagePackBackingStore(file); - const envelope = { key: "k", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }; + const envelope = { key: "k", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; await assert.rejects(() => store.pullAll(), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); await assert.rejects(() => store.push(envelope), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); await assert.rejects(() => store.pushAll([envelope], { soft: true }), (error: NodeJS.ErrnoException) => error.code === "ENOENT"); @@ -1552,7 +1552,7 @@ test("R3 decides resolve-or-refuse: a push through a live link replaces the link const volume = join(dir, "volume"); await mkdir(volume); const target = join(volume, "store.cc"); - const envelope = { key: "a", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }; + const envelope = { key: "a", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }; await new SingleFileMessagePackBackingStore(target).push(envelope); const before = await readFile(target); const link = join(dir, "store.cc"); @@ -1568,7 +1568,7 @@ test("R3 decides resolve-or-refuse: a pushAll through a dangling link replaces t await mkdir(volume); const link = join(dir, "store.cc"); await symlink(join(volume, "store.cc"), link); - await new SingleFileMessagePackBackingStore(link).pushAll([{ key: "a", type: "t", payload: Uint8Array.of(0x90), storedAt: "2026-09-30T00:00:00Z" }]); + await new SingleFileMessagePackBackingStore(link).pushAll([{ key: "a", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }]); assert.ok((await lstat(link)).isFile()); assert.deepEqual(await readdir(volume), []); }); @@ -1585,6 +1585,6 @@ test("CultCache inspector refuses the same vectors by name", async () => { test("a v1 store written at the base commit still reads byte for byte", async () => { const envelopes = await pullVector("v1-base.msgpack"); assert.deepEqual(envelopes.map((entry) => [entry.key, entry.type]), [["alpha", "vectors.item"], ["beta", "vectors.item"]]); - assert.deepEqual([...envelopes[0]!.payload], [0x92, 0xa5, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x01]); - assert.deepEqual([...envelopes[1]!.payload], [0x92, 0xa4, 0x62, 0x65, 0x74, 0x61, 0x02]); + assert.deepEqual([...documentPayloadBytes(envelopes[0]!.payload)], [0x92, 0xa5, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x01]); + assert.deepEqual([...documentPayloadBytes(envelopes[1]!.payload)], [0x92, 0xa4, 0x62, 0x65, 0x74, 0x61, 0x02]); }); diff --git a/packages/cultcache-ts/test/decode-document-payload.test.ts b/packages/cultcache-ts/test/decode-document-payload.test.ts index e212c54a..a18055de 100644 --- a/packages/cultcache-ts/test/decode-document-payload.test.ts +++ b/packages/cultcache-ts/test/decode-document-payload.test.ts @@ -4,8 +4,9 @@ import { test } from "node:test"; import { encode } from "@msgpack/msgpack"; import { z } from "zod"; -import { codecs, decodeDocumentPayload, defineDocumentType } from "../src"; +import { codecs, decodeDocumentPayload, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload } from "../src"; +const payloadOf = (bytes: Uint8Array) => documentPayloadFromBytes(bytes); const note = z.object({ id: z.string(), count: z.number() }); const plain = defineDocumentType({ type: "t.plain", schemaId: "t.plain.v0", schema: note, name: "id" }); const codec = defineDocumentType({ @@ -24,13 +25,19 @@ const formatted = defineDocumentType({ }); test("a payload is read by its definition's formatter, a codec's, or a plain map", () => { - assert.deepEqual(decodeDocumentPayload(plain, encode({ id: "a", count: 1 })), { id: "a", count: 1 }); - assert.deepEqual(decodeDocumentPayload(codec, encode(["a", 1])), { id: "a", count: 1 }); - assert.deepEqual(decodeDocumentPayload(formatted, new Uint8Array([0xc0])), { id: "from-formatter", count: 0 }); + assert.deepEqual(decodeDocumentPayload(plain, payloadOf(encode({ id: "a", count: 1 }))), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(codec, payloadOf(encode(["a", 1]))), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(formatted, payloadOf(new Uint8Array([0xc0]))), { id: "from-formatter", count: 0 }); }); test("a definition that declares both a codec and a formatter is refused where its payload is read", () => { const both = { ...codec, formatter: formatted.formatter } as typeof codec; - assert.throws(() => decodeDocumentPayload(both, encode(["a", 1])), (error: Error) => + assert.throws(() => decodeDocumentPayload(both, payloadOf(encode(["a", 1]))), (error: Error) => error.message === 'CultCache document type "t.codec" declares both a codec and a formatter.'); }); + +test("a payload is written by its definition's codec or formatter, the way it is read", () => { + assert.deepEqual(documentPayloadBytes(encodeDocumentPayload(codec, { id: "a", count: 1 })), encode(["a", 1])); + assert.deepEqual(documentPayloadBytes(encodeDocumentPayload(plain, { id: "a", count: 1 })), encode({ id: "a", count: 1 })); + assert.deepEqual(documentPayloadBytes(encodeDocumentPayload(formatted, { id: "a", count: 1 })), encode(["a"])); +}); diff --git a/packages/cultcache-ts/test/payload-type.test.ts b/packages/cultcache-ts/test/payload-type.test.ts new file mode 100644 index 00000000..7d7e31f7 --- /dev/null +++ b/packages/cultcache-ts/test/payload-type.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { decode, encode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { + decodeDocumentPayload, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, + type AnyCultCacheDocumentDefinition, + type CultCacheEnvelope, + type DocumentPayload, +} from "../src"; + +// The rule: a document payload is read only through its definition. The type is the enforcement, so this file is the +// proof: tsc compiles it only if every line below that carries @ts-expect-error is an error. A line that starts to +// compile makes its @ts-expect-error unused, which is itself an error, so the suite stops until the payload is opaque +// again. The function is never called; the lines are not meant to run. +export function refusedReaders(envelope: CultCacheEnvelope, definition: AnyCultCacheDocumentDefinition, payload: DocumentPayload): void { + // @ts-expect-error a plain msgpack decoder does not accept a document payload + decode(envelope.payload); + // @ts-expect-error a payload is not assignable to bytes + const bytes: Uint8Array = envelope.payload; + // @ts-expect-error nor is it a byte source a copy may be taken from + new Uint8Array(envelope.payload); + // @ts-expect-error a definition's formatter reads bytes, not a payload; the owner unwraps for it + definition.formatter!.decode(envelope.payload); + // @ts-expect-error the owner needs the definition that says how the bytes are laid out + decodeDocumentPayload(undefined, payload); + // @ts-expect-error bytes are not a payload until they enter by name + const forged: DocumentPayload = new Uint8Array(); + void [bytes, forged]; +} + +const note = defineDocumentType({ + type: "t.note", + schemaId: "t.note.v0", + schema: z.object({ id: z.string(), count: z.number() }), + name: "id", +}); + +test("a payload is the same bytes at run time, so frames and stores see no change", () => { + const bytes = encode({ id: "a", count: 1 }); + const payload = documentPayloadFromBytes(bytes); + assert.equal(documentPayloadBytes(payload), bytes); + assert.equal(payload.byteLength, bytes.byteLength); +}); + +test("the owner writes a payload through the definition's schema and reads it back through the same one", () => { + const payload = encodeDocumentPayload(note, { id: "a", count: 1 }); + assert.deepEqual(decode(documentPayloadBytes(payload)), { id: "a", count: 1 }); + assert.deepEqual(decodeDocumentPayload(note, payload), { id: "a", count: 1 }); + // A refusal names the field and never echoes the value it refused. + const refusesWithoutEchoing = (action: () => unknown) => + assert.throws(action, (error: Error) => /count/.test(error.message) && !error.message.includes("canary-value")); + refusesWithoutEchoing(() => encodeDocumentPayload(note, { id: "a", count: "canary-value" })); + refusesWithoutEchoing(() => decodeDocumentPayload(note, documentPayloadFromBytes(encode({ id: "a", count: "canary-value" })))); +}); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 5fa663b3..1502b83d 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -12,7 +12,7 @@ import { codecs } from "../src"; import { MAX_SLOT_KEY, type Codec } from "../src/codec"; import { CultCache } from "../src/cult-cache"; import { readWire, WireFloat, writeWire } from "../src/wire"; -import { defineDocumentType } from "../src/document"; +import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; @@ -195,7 +195,7 @@ test("union-surface.cc reads through SingleFileMessagePackBackingStore and write const file = join(dir, "union-surface.cc"); await writeFile(file, await read("union-surface.cc")); const [envelope] = await new SingleFileMessagePackBackingStore(file).pullAll(); - assert.equal(hex(envelope.payload), hex(await read("union-surface.msgpack"))); + assert.equal(hex(documentPayloadBytes(envelope.payload)), hex(await read("union-surface.msgpack"))); const document = surfaceDocument(envelope.schemaId); const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); @@ -206,7 +206,7 @@ test("union-surface.cc reads through SingleFileMessagePackBackingStore and write const fresh = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(written)).build(); await fresh.put(document, envelope.key, golden); const [rewritten] = await new SingleFileMessagePackBackingStore(written).pullAll(); - assert.equal(hex(rewritten.payload), hex(envelope.payload)); + assert.equal(hex(documentPayloadBytes(rewritten.payload)), hex(documentPayloadBytes(envelope.payload))); // The reference refuses a store whose schema catalog names another type for a slot, so the catalog TS writes must be // the one C# wrote: same canonical schema text, same member names, slots and types. @@ -418,13 +418,13 @@ test("schema.parse still runs on a decoded value, and a document without a codec schema: z.object({ a: z.string().min(3) }), codec: slots<{ a: string }>({ 0: ["a", string] }), }); - const store = memoryStore([{ key: "k", type: "t", payload: encode(["xy"]), storedAt: new Date().toISOString() }]); + const store = memoryStore([{ key: "k", type: "t", payload: documentPayloadFromBytes(encode(["xy"])), storedAt: new Date().toISOString() }]); await assert.rejects(() => CultCache.builder().withDocumentType(strict).withGenericStore(store).build().pullAllBackingStores()); const plain = defineDocumentType({ type: "p", schema: z.object({ id: z.string() }) }); const holder = memoryStore([]); await CultCache.builder().withDocumentType(plain).withGenericStore(holder).build().put(plain, "k", { id: "d" }); - assert.deepEqual(decode(holder.entries[0].payload), { id: "d" }); + assert.deepEqual(decode(documentPayloadBytes(holder.entries[0].payload)), { id: "d" }); }); test("bytes and nullable round-trip through a slot record", () => { @@ -508,7 +508,7 @@ test("a definition that skips defineDocumentType still writes its codec's bytes const holder = memoryStore([]); const cache = CultCache.builder().withDocumentType(raw).withGenericStore(holder).build(); await cache.put(raw as never, "surface", golden as never); - assert.equal(hex(holder.entries[0].payload), hex(await read("union-surface.msgpack"))); + assert.equal(hex(documentPayloadBytes(holder.entries[0].payload)), hex(await read("union-surface.msgpack"))); const [snapshot] = cache.snapshot(); assert.deepEqual(snapshot.catalogEntry!.members!.map((member) => member.typeName).slice(0, 2), ["System.String", "System.Int32"]); }); @@ -687,7 +687,7 @@ test("the Int32 edges of an arm key: the largest is an arm, one past either end }); function envelopeOf(payload: Uint8Array): CultCacheEnvelope { - return { key: "surface", type: "tests.union_surface", payload, storedAt: new Date().toISOString() }; + return { key: "surface", type: "tests.union_surface", payload: documentPayloadFromBytes(payload), storedAt: new Date().toISOString() }; } function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } { diff --git a/packages/cultmesh-browser/README.md b/packages/cultmesh-browser/README.md index 59a44016..58aef6d2 100644 --- a/packages/cultmesh-browser/README.md +++ b/packages/cultmesh-browser/README.md @@ -27,9 +27,17 @@ const counter = await mesh.leaseRawDocument({ recordKey: "counter:main", }); -counter.watch(record => renderCounter(decodeCultNetPayload(record))); +counter.watch(record => renderCounter(decodeCultNetPayload(record, counterDocument))); ``` +A record's `payload` is a `DocumentPayload`, an opaque type over the same bytes, and +`decodeCultNetPayload(record, definition)` is how it is read: through the CultCache document +definition that says whether the bytes are a plain map, a codec's slot record or a +formatter's, then through the definition's schema. A plain MessagePack decode of a payload +does not compile, and `decodeCultNetPayload` takes no definition-less form. The definition +comes from `@gamecult/cultcache-ts` (this package imports only its `./document` entry, which +holds no Node built-ins). + `invoke(...)` returns the correlated provider response. Routing, schema, and envelope failures reject immediately with `CultMeshBrowserOperationError`; callers can branch on its stable `status` and `code` without parsing prose or diff --git a/packages/cultmesh-browser/package.json b/packages/cultmesh-browser/package.json index e35f143f..cd381854 100644 --- a/packages/cultmesh-browser/package.json +++ b/packages/cultmesh-browser/package.json @@ -40,6 +40,7 @@ "node": ">=20" }, "dependencies": { + "@gamecult/cultcache-ts": "^0.14.0", "cultnet-ts": "^0.2.0" }, "devDependencies": { diff --git a/packages/cultmesh-browser/src/index.ts b/packages/cultmesh-browser/src/index.ts index 2aaca7bf..5c20806c 100644 --- a/packages/cultmesh-browser/src/index.ts +++ b/packages/cultmesh-browser/src/index.ts @@ -1,3 +1,5 @@ +import { decodeDocumentPayload } from "@gamecult/cultcache-ts/document"; +import type { AnyCultCacheDocumentDefinition, CultCacheDocumentValue } from "@gamecult/cultcache-ts"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { encodeCultNetMessageForWire, @@ -761,18 +763,18 @@ export class CultMeshBrowserClient implements AsyncDisposable { } /** - * Reads a document record's payload. A document defined with a codec or a formatter is not a plain msgpack map: pass the - * formatter that reads it (a cultcache-ts `resolveDocumentFormatter(definition)`); without one the payload is read as a - * plain value, and a codec document comes back as its positional array. + * Reads a document record's payload through the definition that says how its bytes are laid out: a codec, a formatter or + * a plain map, then the definition's schema. The payload is opaque (`DocumentPayload`), so a plain msgpack decode of it + * does not compile; the definition is required. */ -export function decodeCultNetPayload( +export function decodeCultNetPayload( record: CultNetRawDocumentRecord, - formatter?: { decode(payload: Uint8Array): unknown }, -): T { + definition: TDefinition, +): CultCacheDocumentValue { if (record.payloadEncoding !== "messagepack") { throw new Error(`Unsupported CultNet document payload '${record.payloadEncoding}'.`); } - return (formatter ? formatter.decode(record.payload) : decodeWireValue(record.payload)) as T; + return decodeDocumentPayload(definition, record.payload); } function recordMatchesSchema(record: CultNetRawDocumentRecord, requestedSchema: string): boolean { diff --git a/packages/cultmesh-browser/test/cultmesh-browser.test.ts b/packages/cultmesh-browser/test/cultmesh-browser.test.ts index fb65fbf3..1206b32d 100644 --- a/packages/cultmesh-browser/test/cultmesh-browser.test.ts +++ b/packages/cultmesh-browser/test/cultmesh-browser.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { codecs, defineDocumentType, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { encodeCultNetMessageForWire, @@ -120,6 +121,13 @@ test("Odin rendezvous rejects a poisoned route without vetoing a certified fallb assert.equal(selected.endpoint, signed.route.endpoint); }); +// The counter document the fake provider serves: a plain map. +const counterDocument = defineDocumentType({ + type: "sample.counter_state", + schemaId: "sample.counter_state.v1", + schema: { parse: (value: unknown) => value as { count: number } }, +}); + test("browser client leases provider state, invokes an operation, and follows route replacement", async () => { const first = new FakeProvider("ws://127.0.0.1:4101/mesh", 4); const second = new FakeProvider("ws://127.0.0.1:4102/mesh", 9); @@ -139,11 +147,11 @@ test("browser client leases provider state, invokes an operation, and follows ro schemaId: "sample.counter_state.v1", recordKey: "counter:main", }); - assert.deepEqual(decodeCultNetPayload<{ count: number }>(lease.current!).count, 4); + assert.deepEqual(decodeCultNetPayload(lease.current!, counterDocument).count, 4); const observed: number[] = []; const unwatch = lease.watch(record => { - if (record) observed.push(decodeCultNetPayload<{ count: number }>(record).count); + if (record) observed.push(decodeCultNetPayload(record, counterDocument).count); }); const response = await client.invoke({ serviceId: "sample.counter", @@ -158,7 +166,7 @@ test("browser client leases provider state, invokes an operation, and follows ro route = second.route; await client.refreshRoute(); assert.equal(client.state, "connected"); - assert.deepEqual(decodeCultNetPayload<{ count: number }>(lease.current!).count, 9); + assert.deepEqual(decodeCultNetPayload(lease.current!, counterDocument).count, 9); assert.deepEqual(observed, [4, 6, 9]); assert.equal(first.activeSubscriptionCount, 0); assert.equal(second.activeSubscriptionCount, 1); @@ -698,7 +706,7 @@ class FakeProvider { recordKey: "counter:main", storedAt: "2026-08-17T00:00:00Z", payloadEncoding: "messagepack", - payload: encodeWireValue({ count: this.#count }), + payload: documentPayloadFromBytes(encodeWireValue({ count: this.#count })), sourceRuntimeId: this.#runtimeId, }; } @@ -976,14 +984,25 @@ function base64ToBytes(value: string): Uint8Array { return Uint8Array.from(binary, character => character.charCodeAt(0)); } -test("decodeCultNetPayload reads a payload through the formatter it is given, and plain without one", () => { - const record = { schemaId: "t.codec.v0", recordKey: "k", payloadEncoding: "messagepack", payload: encodeWireValue([7]) } as unknown as CultNetRawDocumentRecord; - const formatter = { decode: (payload: Uint8Array) => ({ count: (decodeWireValue(payload) as number[])[0] }) }; - assert.deepEqual(decodeCultNetPayload<{ count: number }>(record, formatter), { count: 7 }); - assert.deepEqual(decodeCultNetPayload(record), [7]); +test("decodeCultNetPayload reads a payload through the definition it is given: plain map, codec or formatter", () => { + const record = (bytes: Uint8Array) => ({ + schemaId: "t.v0", recordKey: "k", payloadEncoding: "messagepack", payload: documentPayloadFromBytes(bytes), + }) as unknown as CultNetRawDocumentRecord; + const schema = { parse: (value: unknown) => value as { count: number } }; + const plain = defineDocumentType({ type: "t.plain", schemaId: "t.plain.v0", schema }); + const codec = defineDocumentType({ + type: "t.codec", schemaId: "t.codec.v0", schema, codec: codecs.slots<{ count: number }>({ 0: ["count", codecs.int32] }, "T.Count"), + }); + const formatted = defineDocumentType({ + type: "t.formatted", schemaId: "t.formatted.v0", schema, + formatter: { encode: value => encodeWireValue([value.count]), decode: payload => ({ count: (decodeWireValue(payload) as number[])[0]! }) }, + }); + assert.deepEqual(decodeCultNetPayload(record(encodeWireValue({ count: 7 })), plain), { count: 7 }); + assert.deepEqual(decodeCultNetPayload(record(encodeWireValue([7])), codec), { count: 7 }); + assert.deepEqual(decodeCultNetPayload(record(encodeWireValue([7])), formatted), { count: 7 }); }); test("decodeCultNetPayload refuses a payload that is not messagepack, naming the encoding and not the payload", () => { - const record = { schemaId: "t.v0", recordKey: "k", payloadEncoding: "json", payload: encodeWireValue("secret-value") } as unknown as CultNetRawDocumentRecord; - assert.throws(() => decodeCultNetPayload(record), (error: Error) => error.message === "Unsupported CultNet document payload 'json'."); + const record = { schemaId: "t.v0", recordKey: "k", payloadEncoding: "json", payload: documentPayloadFromBytes(encodeWireValue("secret-value")) } as unknown as CultNetRawDocumentRecord; + assert.throws(() => decodeCultNetPayload(record, counterDocument), (error: Error) => error.message === "Unsupported CultNet document payload 'json'."); }); diff --git a/packages/cultmesh-browser/test/payload-type.test.ts b/packages/cultmesh-browser/test/payload-type.test.ts new file mode 100644 index 00000000..9cc95632 --- /dev/null +++ b/packages/cultmesh-browser/test/payload-type.test.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { decodeWireValue } from "cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "cultnet-ts/contracts"; + +import { decodeCultNetPayload } from "../src/index.js"; + +// The rule: a document payload is read only through its definition. A CultNet record's payload is the opaque +// DocumentPayload, so tsc compiles this file only if each line carrying an expect-error directive is an error. The function is +// never called. +export function refusedReaders(record: CultNetRawDocumentRecord): void { + // @ts-expect-error the wire value decoder is for frames and values, not for a document's payload + decodeWireValue(record.payload); + // @ts-expect-error a record's payload is not assignable to bytes + const bytes: Uint8Array = record.payload; + // @ts-expect-error the browser reader needs the definition that says how the bytes are laid out + decodeCultNetPayload(record); + // @ts-expect-error bytes are not a payload until they enter by name + const forged: DocumentPayload = new Uint8Array(); + void [bytes, forged]; +} + +test("a record's payload is the same bytes it entered with by name", () => { + const bytes = Uint8Array.from([0x90]); + assert.equal(documentPayloadBytes(documentPayloadFromBytes(bytes)), bytes); +}); diff --git a/packages/cultmesh-ts/README.md b/packages/cultmesh-ts/README.md index d3fe565b..e1fa368b 100644 --- a/packages/cultmesh-ts/README.md +++ b/packages/cultmesh-ts/README.md @@ -411,9 +411,11 @@ payload binary normalization, and MessagePack decode behind the same `CultMeshDocumentHandle` shape as local documents. The helper prefers an exact schema id match and falls back to the requested record key, so runtimes that alias the same logical document through different generated schema names can -still read the publication with one CultMesh call. Pass a CultCache document -definition instead of a raw schema id when you want a typed handle whose -`latest()` result is parsed through that definition. +still read the publication with one CultMesh call. The handle takes a CultCache +document definition, and its `latest()` result is read and parsed through that +definition: a payload is a `DocumentPayload`, which only +`decodeDocumentPayload(definition, payload)` reads, so no document is read +without the definition that says how its bytes are laid out. Use `node.syncDocumentFromPeerSnapshot(...)` or `CultMesh.syncDocumentFromPeerSnapshot(...)` when a runtime should hydrate its @@ -424,8 +426,8 @@ keep using `node.document(aliasDefinition, key)` and explicitly select `authoritativeWriter()`, `predictionWriter()`, or `observe()` without re-threading the RUDP peer. -The same typed-definition overload is available for -`CultMesh.documentFromStore(...)` and `CultMesh.documentFromSingleFile(...)`, so +`CultMesh.documentFromStore(...)` and `CultMesh.documentFromSingleFile(...)` take +the same definition, so local schema publications can be read as typed reactive documents without manually pulling records, decoding MessagePack, or parsing the payload. When the caller wants source selection to be configuration, use diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 8dee1a39..48b2f742 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -6076,9 +6076,9 @@ export class CultMesh { const onMessage = (message: CultNetMessage): void => { if (message.schemaVersion !== "cultnet.document_put_raw.v0" || message.messageId !== messageId || message.document.schemaId !== receiptBinding.definition.schemaId) return; try { - const decoded = decodeDocumentPayload(receiptBinding.definition, new Uint8Array(message.document.payload)); + const decoded = decodeDocumentPayload(receiptBinding.definition, message.document.payload); cleanup(); - resolve(receiptBinding.definition.schema.parse(decoded) as CultCacheDocumentValue); + resolve(decoded); } catch (error) { cleanup(); reject(error); } }; const cleanup = (): void => { clearTimeout(timeout); peer.off("message", onMessage); }; diff --git a/packages/cultmesh-ts/src/provider-rudp-transport.ts b/packages/cultmesh-ts/src/provider-rudp-transport.ts index 7e94df02..fc0d75c0 100644 --- a/packages/cultmesh-ts/src/provider-rudp-transport.ts +++ b/packages/cultmesh-ts/src/provider-rudp-transport.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { createSocket, type Socket } from "node:dgram"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { encodeWireValue } from "cultnet-ts/wire"; import { CultNetPeer, @@ -171,7 +172,9 @@ class RudpProviderConnection implements CultMeshProviderConnection { recordKey: publication.recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue(publication.value), + // The one plain writer of a document payload: a publication carries no definition. The typed owner of this write is + // cultlib-gaps:follow_up:document-payload-encode-owner. + payload: documentPayloadFromBytes(encodeWireValue(publication.value)), sourceRuntimeId: this.#identity.serviceInstanceId, sourceAgentId: this.#identity.providerId, sourceRole: "provider", diff --git a/packages/cultmesh-ts/src/provider-session-broker.ts b/packages/cultmesh-ts/src/provider-session-broker.ts index 8f650e36..75240a14 100644 --- a/packages/cultmesh-ts/src/provider-session-broker.ts +++ b/packages/cultmesh-ts/src/provider-session-broker.ts @@ -1,3 +1,4 @@ +import { encodeDocumentPayload } from "@gamecult/cultcache-ts"; import { encodeWireValue } from "cultnet-ts/wire"; import type { CultNetDocumentPutRawMessage, @@ -13,6 +14,7 @@ import { cultMeshProviderSessionSchemas, decodeProviderSessionPayload, encodeProviderSessionPayload, + providerCommandDocument, type CultMeshProviderCommandReceiptWire, type CultMeshProviderCommandWire, type CultMeshProviderLeaseRenewalWire, @@ -420,7 +422,7 @@ export class CultMeshProviderSessionBroker { recordKey: `${command.providerId}/${command.serviceInstanceId}/${command.commandId}`, storedAt: this.#clock.now().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue(command), + payload: encodeDocumentPayload(providerCommandDocument, command), sourceRuntimeId: this.#options.runtimeId, sourceRole: "provider-session-broker", }, diff --git a/packages/cultmesh-ts/src/provider-session-wire.ts b/packages/cultmesh-ts/src/provider-session-wire.ts index 9bda76a6..d4884694 100644 --- a/packages/cultmesh-ts/src/provider-session-wire.ts +++ b/packages/cultmesh-ts/src/provider-session-wire.ts @@ -1,3 +1,4 @@ +import { decodeDocumentPayload, defineDocumentType } from "@gamecult/cultcache-ts"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import type { CultNetRawDocumentRecord } from "cultnet-ts"; @@ -155,6 +156,23 @@ export function assertProviderConnectEvidence( } } +// The provider command is a CultCache document like any other: written with encodeDocumentPayload and read with +// decodeDocumentPayload, so its bytes never pass through a plain decoder. +export const providerCommandDocument = defineDocumentType({ + type: cultMeshProviderSessionSchemas.command, + schemaId: cultMeshProviderSessionSchemas.command, + schema: { + parse(value: unknown): CultMeshProviderCommandWire { + const command = requireObject(value, "provider command"); + requireText(command.commandId, "commandId"); + requireText(command.commandKind, "commandKind"); + requireText(command.providerId, "providerId"); + requireText(command.serviceInstanceId, "serviceInstanceId"); + return command as unknown as CultMeshProviderCommandWire; + }, + }, +}); + export function decodeProviderCommandDocument(document: CultNetRawDocumentRecord): CultMeshProviderCommandWire { if (document.schemaId !== cultMeshProviderSessionSchemas.command) { throw new Error(`Expected ${cultMeshProviderSessionSchemas.command}, received ${document.schemaId}.`); @@ -162,12 +180,7 @@ export function decodeProviderCommandDocument(document: CultNetRawDocumentRecord if (document.payloadEncoding !== "messagepack") { throw new Error(`Provider command ${document.recordKey} must use MessagePack.`); } - const command = decodeWireValue(document.payload) as Partial; - requireText(command.commandId, "commandId"); - requireText(command.commandKind, "commandKind"); - requireText(command.providerId, "providerId"); - requireText(command.serviceInstanceId, "serviceInstanceId"); - return command as CultMeshProviderCommandWire; + return decodeDocumentPayload(providerCommandDocument, document.payload); } export function assertProviderLease(value: unknown): asserts value is CultMeshProviderLeaseWire { diff --git a/packages/cultmesh-ts/src/realtime-wire.ts b/packages/cultmesh-ts/src/realtime-wire.ts index fa647133..9fc99789 100644 --- a/packages/cultmesh-ts/src/realtime-wire.ts +++ b/packages/cultmesh-ts/src/realtime-wire.ts @@ -15,6 +15,7 @@ // the same C# test decodes. A fixture authored on one side alone would only // agree with itself. +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; import { isNullOrWhiteSpaceCSharp } from "cultnet-ts"; /** `CultMeshRealtimeWireProtocol.ApplicationProtocolName`: the QUIC ALPN. */ @@ -55,7 +56,7 @@ export interface CultMeshRealtimeFrame { producerEpoch: bigint; sequence: bigint; delivery: CultMeshRealtimeDelivery; - payload: Uint8Array; + payload: DocumentPayload; } const MAGIC = 0x31545343; @@ -123,7 +124,7 @@ export function encodeRealtimeFrame(frame: CultMeshRealtimeFrame): Uint8Array { result.set(channel, offset); offset += channel.byteLength; result.set(schema, offset); offset += schema.byteLength; result.set(body, offset); offset += body.byteLength; - result.set(frame.payload, offset); + result.set(documentPayloadBytes(frame.payload), offset); return result; } @@ -186,6 +187,6 @@ export function decodeRealtimeFrame(bytes: Uint8Array): CultMeshRealtimeFrame { // would compare equal or not depending on how it reached the decoder. // // The constructor copies and is the one type on both inputs. - payload: new Uint8Array(bytes.subarray(offset, offset + payloadLength)), + payload: documentPayloadFromBytes(new Uint8Array(bytes.subarray(offset, offset + payloadLength))), }; } diff --git a/packages/cultmesh-ts/test/codec-documents.test.ts b/packages/cultmesh-ts/test/codec-documents.test.ts index aeb2a14d..57be7706 100644 --- a/packages/cultmesh-ts/test/codec-documents.test.ts +++ b/packages/cultmesh-ts/test/codec-documents.test.ts @@ -136,3 +136,41 @@ test("the mesh syncs a codec document from a peer snapshot and returns a codec r server.close(); } }); + +test("a RUDP put naming a schema the server has no definition for is refused before the handler, and the refusal does not echo the schema id", async () => { + const connectionId = 0x10203070; + const canary = "cultmesh.canary_schema_8c1f.v0"; + const stray = defineDocumentType({ + type: "cultmesh.canary_schema", + schemaId: canary, + schema: z.object({ noteId: z.string() }), + name: "noteId", + }); + const known = defineCultNetDocumentBinding({ definition: plainNote }); + const errors: Error[] = []; + const puts: unknown[] = []; + const server = CultMesh.createRudpDocumentServer("cultmesh-ts-unregistered-server", connectionId, { + documents: new CultNetDocumentRegistry([known]), + bindHost: "127.0.0.1", bindPort: 0, resendDelayMs: 25, resendPollMs: 5, + onError: (error) => errors.push(error), + onDocumentPutRaw: async (document) => { puts.push(document.payload); }, + }); + let peer: CultNetPeer | undefined; + try { + await server.start(); + peer = await CultMesh.createRudpPeer("cultmesh-ts-unregistered-client", connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 25, resendPollMs: 5, connectTimeoutMs: 1_000, + }); + peer.send(new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: stray })]) + .createRawDocumentPutMessage(defineCultNetDocumentBinding({ definition: stray }), "unregistered-1", "k", { noteId: "k" })); + const startedAt = Date.now(); + while (errors.length === 0 && Date.now() - startedAt < 2_000) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(errors.length, 1); + assert.match(errors[0]!.message, /no registered definition/); + assert.ok(!errors[0]!.message.includes(canary), "the refusal names no schema id"); + assert.deepEqual(puts, [], "the handler never saw the bytes"); + } finally { + peer?.close(); + server.close(); + } +}); diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index e1c4ed49..6c972731 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -6,7 +6,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; import { z } from "zod"; -import { defineDocumentType } from "@gamecult/cultcache-ts"; +import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetPeer, @@ -22,6 +22,9 @@ import { } from "cultnet-ts"; import { CultMesh, CultMeshRudpUnservableDocumentError } from "../src/index"; +// A record's payload as the bytes a peer would put on the wire. +const payloadOf = (value: unknown) => documentPayloadFromBytes(encodeWireValue(value)); + const noteDocument = defineDocumentType({ type: "cultmesh.note", schemaId: "cultmesh.note.v0", @@ -268,7 +271,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa type: "runtime.generated.incompatible-note", schemaId: "runtime.generated.incompatible-note.v1", storedAt: "2026-06-27T00:00:00Z", - payload: encodeWireValue({ nope: "not the requested shape" }), + payload: payloadOf({ nope: "not the requested shape" }), catalogEntry: { schemaId: "runtime.generated.incompatible-note.v1", schemaName: "runtime.generated.incompatible-note", @@ -283,7 +286,7 @@ test("CultMesh TS store document handles choose compatible foreign schema payloa type: "runtime.generated.cultmesh.note", schemaId: "runtime.generated.cultmesh.note.ui.42", storedAt: "2026-06-27T00:00:01Z", - payload: encodeWireValue({ + payload: payloadOf({ noteId: "note:foreign", body: "foreign schema id, local store shape", }), @@ -2820,7 +2823,7 @@ test("CultMesh TS RUDP document server accepts raw document puts from multiple p recordKey: "note:a", storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue("first"), + payload: payloadOf("first"), sourceRuntimeId: "client-a", }, }); @@ -2832,7 +2835,7 @@ test("CultMesh TS RUDP document server accepts raw document puts from multiple p recordKey: "note:b", storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue("second"), + payload: payloadOf("second"), sourceRuntimeId: "client-b", }, }); @@ -3316,7 +3319,7 @@ function notePut( recordKey, storedAt, payloadEncoding: "messagepack", - payload: encodeWireValue({ noteId: recordKey, body: "x".repeat(bodyLength) }), + payload: payloadOf({ noteId: recordKey, body: "x".repeat(bodyLength) }), ...(sourceRuntimeId ? { sourceRuntimeId } : {}), }, }; @@ -3350,7 +3353,7 @@ function echoReceiptBytes(put: CultNetDocumentPutRawMessage): number { recordKey: put.document.recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: encodeWireValue(decodeWireValue(put.document.payload)), + payload: payloadOf(decodeWireValue(documentPayloadBytes(put.document.payload))), sourceRuntimeId: SERVED_BOUND_SERVER, sourceAgentId: undefined, sourceRole: "provider", @@ -3402,7 +3405,7 @@ async function withServedBoundRig( recordKey: document.recordKey, storedAt: document.storedAt, payloadEncoding: "messagepack", - payload: encodeWireValue(document.payload), + payload: payloadOf(document.payload), }, }); if (!handler.receipt) return undefined; @@ -3453,7 +3456,7 @@ function servedBodies(reply: CultNetMessage): Array<[string, number]> { assert.equal(reply.schemaVersion, "cultnet.snapshot_response_raw.v0"); return (reply as CultNetSnapshotResponseRawMessage).documents.map((document) => [ document.recordKey, - (decodeWireValue(document.payload) as { body: string }).body.length, + (decodeWireValue(documentPayloadBytes(document.payload)) as { body: string }).body.length, ]); } diff --git a/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts b/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts index aadd1265..e7e04c1a 100644 --- a/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts +++ b/packages/cultmesh-ts/test/interop/cultmesh-quic-peer.ts @@ -27,6 +27,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import process from "node:process"; +import { documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { CultMeshQuicRealtimeConnector, CultMeshQuicRealtimeProvider, @@ -124,7 +126,7 @@ async function serveAsync(args: Map): Promise { producerEpoch: 1n, sequence: BigInt(sequence), delivery, - payload: Buffer.from(`frame-${sequence}`, "utf8"), + payload: documentPayloadFromBytes(Buffer.from(`frame-${sequence}`, "utf8")), }; await provider.broadcast(frame); // Printed so the harness can compare this exact frame's TypeScript @@ -215,7 +217,7 @@ async function main(): Promise { producerEpoch: frame.producerEpoch.toString(), sequence: frame.sequence.toString(), delivery: frame.delivery, - payloadHex: Buffer.from(frame.payload).toString("hex"), + payloadHex: Buffer.from(documentPayloadBytes(frame.payload)).toString("hex"), }); } finally { clearTimeout(timer); diff --git a/packages/cultmesh-ts/test/payload-type.test.ts b/packages/cultmesh-ts/test/payload-type.test.ts new file mode 100644 index 00000000..21ee2787 --- /dev/null +++ b/packages/cultmesh-ts/test/payload-type.test.ts @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { decodeWireValue } from "cultnet-ts/wire"; +import type { CultNetRawDocumentRecord } from "cultnet-ts"; + +import { decodeRealtimeFrame, encodeRealtimeFrame, type CultMeshRealtimeFrame } from "../src/realtime-wire"; + +// The rule: a document payload is read only through its definition. Every payload this package touches (a CultNet +// record's, a realtime frame's) is the opaque DocumentPayload, so tsc compiles this file only if each line carrying +// an expect-error directive is an error. The function is never called. +export function refusedReaders(record: CultNetRawDocumentRecord, frame: CultMeshRealtimeFrame): void { + // @ts-expect-error the wire value decoder is for frames and values, not for a document's payload + decodeWireValue(record.payload); + // @ts-expect-error a frame's payload is not assignable to bytes + const bytes: Uint8Array = frame.payload; + // @ts-expect-error nor may a copy of it be taken as bytes + new Uint8Array(record.payload); + // @ts-expect-error bytes are not a payload until they enter by name + const forged: DocumentPayload = new Uint8Array(); + void [bytes, forged]; +} + +test("a realtime frame carries its payload as the same bytes it was given by name", () => { + const bytes = Uint8Array.from([1, 2, 3]); + const frame: CultMeshRealtimeFrame = { + channelId: "c", schemaId: "s", bodyId: "b", producerEpoch: 1n, sequence: 1n, delivery: "reliable-ordered", + payload: documentPayloadFromBytes(bytes), + }; + assert.deepEqual(documentPayloadBytes(decodeRealtimeFrame(encodeRealtimeFrame(frame)).payload), bytes); +}); diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index abb8f010..e9a9f1c5 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -1,6 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { createSocket } from "node:dgram"; +import { documentPayloadBytes } from "@gamecult/cultcache-ts"; import { decodeWireValue } from "cultnet-ts/wire"; import { CultNetDocumentRegistry, @@ -46,7 +47,7 @@ test("RUDP provider transport requires broker acceptance for the full lifecycle" session.connectPayload !== undefined && decodeProviderConnectEvidence(session.connectPayload).sessionToken === "signed-provider-token", acceptPublication: (_identity, publicationId, document) => { - accepted.set(publicationId, decodeWireValue(document.payload)); + accepted.set(publicationId, decodeWireValue(documentPayloadBytes(document.payload))); acceptedProvenance.set(publicationId, { runtime: document.sourceRuntimeId, agent: document.sourceAgentId, diff --git a/packages/cultmesh-ts/test/provider-session-wire.test.ts b/packages/cultmesh-ts/test/provider-session-wire.test.ts index ac607163..a9fd3345 100644 --- a/packages/cultmesh-ts/test/provider-session-wire.test.ts +++ b/packages/cultmesh-ts/test/provider-session-wire.test.ts @@ -1,10 +1,16 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload, type DocumentPayload } from "@gamecult/cultcache-ts"; +import { encodeWireValue } from "cultnet-ts/wire"; + import { + cultMeshProviderSessionSchemas, + decodeProviderCommandDocument, decodeProviderConnectEvidence, encodeProviderConnectEvidence, encodeProviderSessionPayload, + providerCommandDocument, } from "../src/provider-session-wire"; const REGISTRATION_GOLDEN = "hqpwcm92aWRlcklkqGFldGhlcmlhsXNlcnZpY2VJbnN0YW5jZUlkq2FldGhlcmlhLTQyqmVuZHBvaW50SWSvYWV0aGVyaWEtcHVibGljp3ZlcnNlSWSmcHVibGljuHJlcXVlc3RlZExlYXNlRHVyYXRpb25Nc811MLBhdXRob3JpdHlMZWFzZUlkq2F1dGhvcml0eS03"; @@ -41,3 +47,21 @@ test("provider Connect evidence keeps transport generation separate from authori TOKENLESS_CONNECT_EVIDENCE_GOLDEN, ); }); + +test("a provider command is a CultCache document: written and read by its definition, and a refusal names no value", () => { + const command = { commandId: "c1", commandKind: "k", providerId: "p", serviceInstanceId: "s", payload: { n: 1 } }; + const payload = encodeDocumentPayload(providerCommandDocument, command); + assert.deepEqual(documentPayloadBytes(payload), encodeWireValue(command), "the wire bytes are the plain map they always were"); + const record = (bytes: DocumentPayload) => ({ + schemaId: cultMeshProviderSessionSchemas.command, recordKey: "p/s/c1", storedAt: "2026-10-11T00:00:00Z", + payloadEncoding: "messagepack" as const, payload: bytes, + }); + assert.deepEqual(decodeProviderCommandDocument(record(payload)), command); + + const refused = documentPayloadFromBytes(encodeWireValue({ ...command, providerId: "canary-provider-5d2e", commandKind: "" })); + assert.throws( + () => decodeProviderCommandDocument(record(refused)), + (error: Error) => /commandKind/.test(error.message) && !error.message.includes("canary-provider-5d2e"), + ); + assert.throws(() => encodeDocumentPayload(providerCommandDocument, { ...command, commandId: " " }), /commandId/); +}); diff --git a/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts b/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts index 8a2063fe..acf06a41 100644 --- a/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts +++ b/packages/cultmesh-ts/test/realtime-quic-consumer.test.ts @@ -14,6 +14,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import test from "node:test"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { canonicalRoute, type CultMeshAuthorityRouteView, @@ -154,7 +156,7 @@ function testFrame(overrides: Partial = {}): CultMeshReal producerEpoch: 7n, sequence: 1n, delivery: "latest-only", - payload: new Uint8Array([3, 4, 5]), + payload: documentPayloadFromBytes(new Uint8Array([3, 4, 5])), ...overrides, }; } diff --git a/packages/cultmesh-ts/test/realtime-quic-provider.test.ts b/packages/cultmesh-ts/test/realtime-quic-provider.test.ts index 8a541641..d174609b 100644 --- a/packages/cultmesh-ts/test/realtime-quic-provider.test.ts +++ b/packages/cultmesh-ts/test/realtime-quic-provider.test.ts @@ -13,6 +13,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import test from "node:test"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { CultMeshQuicNativeRuntime, } from "../src/realtime-quic-native"; @@ -63,7 +65,7 @@ function testFrame(overrides: Partial = {}): CultMeshReal producerEpoch: 7n, sequence: 1n, delivery: "latest-only", - payload: new Uint8Array([3, 4, 5]), + payload: documentPayloadFromBytes(new Uint8Array([3, 4, 5])), ...overrides, }; } @@ -613,8 +615,8 @@ test("a peer attaching after two latest-only broadcasts for one key receives exa if (!nativeBridgeAvailable()) return void t.skip("no native bridge available"); const provider = await startProvider(); try { - await provider.broadcast(testFrame({ sequence: 1n, payload: new Uint8Array([1]) })); - await provider.broadcast(testFrame({ sequence: 2n, payload: new Uint8Array([2]) })); + await provider.broadcast(testFrame({ sequence: 1n, payload: documentPayloadFromBytes(new Uint8Array([1])) })); + await provider.broadcast(testFrame({ sequence: 2n, payload: documentPayloadFromBytes(new Uint8Array([2])) })); // Attach after both broadcasts have already resolved: nothing in this // peer's outbox came from `broadcast()`'s own fan-out, only from the @@ -636,11 +638,11 @@ test("retention keeps the newer generation when an older one broadcasts afterwar if (!nativeBridgeAvailable()) return void t.skip("no native bridge available"); const provider = await startProvider(); try { - await provider.broadcast(testFrame({ sequence: 5n, payload: new Uint8Array([5]) })); + await provider.broadcast(testFrame({ sequence: 5n, payload: documentPayloadFromBytes(new Uint8Array([5])) })); // An older generation broadcast after a newer one must not overwrite the // retained frame: mutating the retention comparison to a plain // last-write-wins assignment would regress this. - await provider.broadcast(testFrame({ sequence: 3n, payload: new Uint8Array([3]) })); + await provider.broadcast(testFrame({ sequence: 3n, payload: documentPayloadFromBytes(new Uint8Array([3])) })); const consumer = await dialProvider(provider); try { @@ -672,7 +674,7 @@ test("a reliable-ordered frame is never retained and is not delivered to a late // after a real latest-only frame, so the assertion below proves the // reliable-ordered frame specifically was never retained, rather than // merely proving the provider had nothing at all to seed. - await provider.broadcast(testFrame({ delivery: "latest-only", sequence: 1n, payload: new Uint8Array([1]) })); + await provider.broadcast(testFrame({ delivery: "latest-only", sequence: 1n, payload: documentPayloadFromBytes(new Uint8Array([1])) })); const secondConsumer = await dialProvider(provider); try { const seeded = await secondConsumer.receiveFrame(); diff --git a/packages/cultmesh-ts/test/realtime-wire.test.ts b/packages/cultmesh-ts/test/realtime-wire.test.ts index 93f23365..d4497b84 100644 --- a/packages/cultmesh-ts/test/realtime-wire.test.ts +++ b/packages/cultmesh-ts/test/realtime-wire.test.ts @@ -16,6 +16,8 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { join } from "node:path"; +import { documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; + import { CULTMESH_REALTIME_ALPN, CULTMESH_REALTIME_CONNECTION_CLOSE_CODE, @@ -89,7 +91,7 @@ function frameOf(fields: VectorFields): CultMeshRealtimeFrame { producerEpoch: 0n, sequence: 0n, delivery: fields.delivery, - payload: payload(fields.payload), + payload: documentPayloadFromBytes(payload(fields.payload)), }; } @@ -175,7 +177,7 @@ const sample = (overrides: Partial = {}): CultMeshRealtim producerEpoch: 1n, sequence: 2n, delivery: "reliable-ordered", - payload: Uint8Array.from([1, 2, 3]), + payload: documentPayloadFromBytes(Uint8Array.from([1, 2, 3])), ...overrides, }); @@ -199,7 +201,7 @@ test("the fixed header is 37 little-endian bytes in the reference's order", () = const encoded = encodeRealtimeFrame(sample({ channelId: "ch", schemaId: "sch", bodyId: "bdy", producerEpoch: 0x0102030405060708n, sequence: 0x1112131415161718n, - delivery: "unreliable", payload: Uint8Array.from([0xaa]), + delivery: "unreliable", payload: documentPayloadFromBytes(Uint8Array.from([0xaa])), })); const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength); assert.equal(view.getUint32(0, true), 0x31545343, "magic, little-endian"); @@ -258,11 +260,11 @@ test("an identity of exactly 65535 UTF-8 bytes encodes and one byte more is refu }); test("a payload at the 64 MiB ceiling encodes and one byte more is refused", () => { - const encoded = encodeRealtimeFrame(sample({ payload: ramp(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES) })); + const encoded = encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(ramp(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES)) })); assert.equal(encoded.byteLength, 37 + 15 + 20 + 6 + CULTMESH_REALTIME_MAX_PAYLOAD_BYTES); assert.equal(decodeRealtimeFrame(encoded).payload.byteLength, CULTMESH_REALTIME_MAX_PAYLOAD_BYTES); assert.throws( - () => encodeRealtimeFrame(sample({ payload: new Uint8Array(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES + 1) })), + () => encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(new Uint8Array(CULTMESH_REALTIME_MAX_PAYLOAD_BYTES + 1)) })), /payload exceeds the QUIC wire limit/, ); }); @@ -363,16 +365,16 @@ test("decode refuses a negative payload length that the length sum cancels out", }); test("decode reads exactly the viewed bytes of an offset view, not the buffer behind it", () => { - const good = encodeRealtimeFrame(sample({ payload: Uint8Array.from([9, 8, 7]) })); + const good = encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(Uint8Array.from([9, 8, 7])) })); const pool = new Uint8Array(good.byteLength + 64).fill(0xa5); pool.set(good, 17); const view = pool.subarray(17, 17 + good.byteLength); - assert.deepEqual(decodeRealtimeFrame(view).payload, Uint8Array.from([9, 8, 7])); + assert.deepEqual(documentPayloadBytes(decodeRealtimeFrame(view).payload), Uint8Array.from([9, 8, 7])); assert.equal(decodeRealtimeFrame(view).channelId, "aetheria.zone-1"); // And the decoded payload does not alias the frame it came from. const decoded = decodeRealtimeFrame(good); good[good.byteLength - 1] = 0; - assert.deepEqual(decoded.payload, Uint8Array.from([9, 8, 7])); + assert.deepEqual(documentPayloadBytes(decoded.payload), Uint8Array.from([9, 8, 7])); }); // The `Uint8Array` case above passes with a plain `bytes.slice(...)`; this one @@ -380,12 +382,12 @@ test("decode reads exactly the viewed bytes of an offset view, not the buffer be // `Buffer` is what a socket read hands a Node caller, so the aliasing bug this // pins is the one that would actually reach a consumer. test("the decoded payload copies out of a Node Buffer too, which overrides slice", () => { - const frame = Buffer.from(encodeRealtimeFrame(sample({ payload: Uint8Array.from([9, 8, 7]) }))); + const frame = Buffer.from(encodeRealtimeFrame(sample({ payload: documentPayloadFromBytes(Uint8Array.from([9, 8, 7])) }))); const decoded = decodeRealtimeFrame(frame); frame[frame.byteLength - 1] = 0; - assert.deepEqual(decoded.payload, Uint8Array.from([9, 8, 7]), "payload unchanged by a later write to the frame"); - assert.equal(Buffer.isBuffer(decoded.payload), false, "the copy is a plain Uint8Array, not a Buffer"); - assert.notEqual(decoded.payload.buffer, frame.buffer, "the copy has its own backing store"); + assert.deepEqual(documentPayloadBytes(decoded.payload), Uint8Array.from([9, 8, 7]), "payload unchanged by a later write to the frame"); + assert.equal(Buffer.isBuffer(documentPayloadBytes(decoded.payload)), false, "the copy is a plain Uint8Array, not a Buffer"); + assert.notEqual(documentPayloadBytes(decoded.payload).buffer, frame.buffer, "the copy has its own backing store"); // The three identities need no such check: `TextDecoder` returns strings, which // hold no reference to the bytes they were decoded from. assert.equal(decoded.channelId, "aetheria.zone-1"); diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts index 8d0425e4..713a3bee 100644 --- a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -5,6 +5,7 @@ import assert from "node:assert/strict"; import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; import test from "node:test"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { encodeWireValue } from "cultnet-ts/wire"; import { CultNetDocumentRegistry, @@ -43,7 +44,7 @@ function put(recordKey: string) { recordKey, storedAt: new Date().toISOString(), payloadEncoding: "messagepack" as const, - payload: encodeWireValue(recordKey), + payload: documentPayloadFromBytes(encodeWireValue(recordKey)), }, }; } diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 4bfdafad..31188790 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -1,3 +1,4 @@ +import type { DocumentPayload } from "@gamecult/cultcache-ts/document"; import Ajv2020, { type ValidateFunction } from "ajv/dist/2020.js"; import documentRecordSchema from "../contracts/cultnet/cultnet.document-record.schema.json"; @@ -200,7 +201,7 @@ export interface CultNetRawDocumentRecord { recordKey: string; storedAt: string; payloadEncoding: CultNetRawPayloadEncoding; - payload: Uint8Array; + payload: DocumentPayload; sourceRuntimeId?: string; sourceAgentId?: string; sourceRole?: string; diff --git a/packages/cultnet-ts/src/idunn-runtime-authority.ts b/packages/cultnet-ts/src/idunn-runtime-authority.ts index 7f7b882f..2ec5456c 100644 --- a/packages/cultnet-ts/src/idunn-runtime-authority.ts +++ b/packages/cultnet-ts/src/idunn-runtime-authority.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { documentPayloadFromBytes } from "@gamecult/cultcache-ts/document"; import { decode, encode } from "@msgpack/msgpack"; import { @@ -278,7 +279,7 @@ export function createIdunnRuntimeSigner(options: IdunnRuntimeSignerOptions): Id recordKey: expected.target, storedAt: new Date().toISOString(), payloadEncoding: "messagepack", - payload: new Uint8Array(signed.payload), + payload: documentPayloadFromBytes(new Uint8Array(signed.payload)), sourceRuntimeId: expected.runtimeId, sourceRole: "runtime-presence-health-publisher", tags: ["cultnet.transport.rudp.v0", "runtime-presence"], diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index 07652497..e12eaf8e 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -5,7 +5,12 @@ import type { CultCacheDocumentValue, CultCacheEnvelope, } from "@gamecult/cultcache-ts"; -import { decodeDocumentPayload, resolveDocumentFormatter } from "@gamecult/cultcache-ts"; +import { + decodeDocumentPayload, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, +} from "@gamecult/cultcache-ts"; import { type CultNetDocumentDeleteMessage, @@ -134,8 +139,6 @@ export class CultNetDocumentRegistry { tags?: string[]; } = {}, ): CultNetDocumentPutRawMessage { - const parsed = binding.definition.schema.parse(value); - const formatter = resolveDocumentFormatter(binding.definition); return { schemaVersion: "cultnet.document_put_raw.v0", messageId, @@ -144,7 +147,7 @@ export class CultNetDocumentRegistry { recordKey: documentKey, storedAt: options.storedAt ?? new Date().toISOString(), payloadEncoding: "messagepack", - payload: formatter.encode(parsed), + payload: encodeDocumentPayload(binding.definition, value), sourceRuntimeId: options.sourceRuntimeId, sourceAgentId: options.sourceAgentId, sourceRole: options.sourceRole, @@ -281,7 +284,7 @@ export class CultNetDocumentRegistry { schemaId: resolution.preserveIncomingSchemaId ? message.document.schemaId : schemaIdForBinding(binding), - payload: new Uint8Array(message.document.payload), + payload: documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(message.document.payload))), storedAt: message.document.storedAt, }, }; @@ -340,7 +343,7 @@ export class CultNetDocumentRegistry { return { binding: exact, preserveIncomingSchemaId: true }; } - const payload = new Uint8Array(document.payload); + const payload = documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(document.payload))); for (const binding of this.#uniqueBindings()) { try { decodeDocumentPayload(binding.definition, payload); @@ -367,7 +370,7 @@ export class CultNetDocumentRegistry { recordKey: envelope.key, storedAt: envelope.storedAt, payloadEncoding: "messagepack", - payload: new Uint8Array(envelope.payload), + payload: documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(envelope.payload))), }; } } diff --git a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts index b5c9236a..2aff6d92 100644 --- a/packages/cultnet-ts/test/idunn-runtime-authority.test.ts +++ b/packages/cultnet-ts/test/idunn-runtime-authority.test.ts @@ -4,6 +4,7 @@ import { spawnSync } from "node:child_process"; import dgram from "node:dgram"; import fs from "node:fs"; import path from "node:path"; +import { documentPayloadBytes, documentPayloadFromBytes, type DocumentPayload } from "@gamecult/cultcache-ts"; import { decode, encode } from "@msgpack/msgpack"; import { @@ -133,18 +134,18 @@ test("rejects a credential file holding more than one envelope", (context) => { ); }); -function presenceSlot(document: { payload: Uint8Array }, slot: number): unknown { - return (decode(document.payload) as unknown[])[slot]; +function presenceSlot(document: { payload: DocumentPayload }, slot: number): unknown { + return (decode(documentPayloadBytes(document.payload)) as unknown[])[slot]; } // Every committed vector is asserted, dead ones included. Stateless sets sign at sequence 1. for (const set of ["web", "service", "rudp-route"] as const) { test(`${set}: a fresh signer answers warming, and after reportHealth('active') answers active, byte for byte as the Rust-verified vectors`, (context) => { freezeClock(context); - assert.deepEqual(Buffer.from(signerFor(context, set).answerRouteObservation(CHALLENGE).payload), fixtureFile(set, "presence-warming.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(signerFor(context, set).answerRouteObservation(CHALLENGE).payload)), fixtureFile(set, "presence-warming.bin")); const active = signerFor(context, set); active.reportHealth("active"); - assert.deepEqual(Buffer.from(active.answerRouteObservation(CHALLENGE).payload), fixtureFile(set, "presence-active.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(active.answerRouteObservation(CHALLENGE).payload)), fixtureFile(set, "presence-active.bin")); }); } @@ -152,7 +153,7 @@ test("a shortfall signs the vector Rust correlates to the typed capacity disagre freezeClock(context); const signer = signerFor(context, "service", -1); signer.reportHealth("active"); - assert.deepEqual(Buffer.from(signer.answerRouteObservation(CHALLENGE).payload), fixtureFile("service", "presence-active-below-minimum.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(signer.answerRouteObservation(CHALLENGE).payload)), fixtureFile("service", "presence-active-below-minimum.bin")); }); test("the signer owns the reported health: a fresh signer is warming, reportHealth moves it, the sequence keeps counting", (context) => { @@ -197,7 +198,7 @@ async function startOdinPeer(context: TestContext, endpoint: string) { context.after(() => peer.close()); const state = { reject: false, - received: [] as Array<{ schemaVersion?: string; document: { payload: Uint8Array } }>, + received: [] as Array<{ schemaVersion?: string; document: { payload: DocumentPayload } }>, responseAck: undefined as number | undefined, failure: undefined as Error | undefined, }; @@ -239,7 +240,7 @@ test("the Odin publisher publishes the signer's document to the endpoint Expecte await publisher.publish("accepted presence"); assert.equal(peer.received[0]?.schemaVersion, "cultnet.document_put_raw.v0"); - const published = decode(peer.received[0]!.document.payload) as unknown[]; + const published = decode(documentPayloadBytes(peer.received[0]!.document.payload)) as unknown[]; assert.equal(published[RUNTIME_PRESENCE_SLOT.state], "warming"); assert.equal(published[RUNTIME_PRESENCE_SLOT.publisherSequence], 1); assert.equal(peer.responseAck, 2, "the response data packet must acknowledge the publisher's reliable schema packet"); @@ -257,7 +258,7 @@ test("the Odin publisher leaves a capability shortfall to the Rust correlation i signer.reportHealth("active"); const publisher = createIdunnRuntimePresencePublisher({ signer, endpoint }); await publisher.publish("below minimum"); - const capabilities = (decode(peer.received[0]!.document.payload) as unknown[])[RUNTIME_PRESENCE_SLOT.capabilities] as unknown[][]; + const capabilities = (decode(documentPayloadBytes(peer.received[0]!.document.payload)) as unknown[])[RUNTIME_PRESENCE_SLOT.capabilities] as unknown[][]; assert.equal(capabilities[0]![3], expectedFacts("service").capabilities[0]!.capacity - 1); }); @@ -304,16 +305,16 @@ function statefulSigner(context: TestContext, lease = "lease.cc") { test("stateful: warming, active and degraded match the Rust-verified vectors byte for byte, active and degraded bound to Rust's lease", (context) => { freezeClock(context); const signer = statefulSigner(context); - assert.deepEqual(Buffer.from(signer.answerRouteObservation(CHALLENGE).payload), fixtureFile(STATEFUL, "presence-warming.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(signer.answerRouteObservation(CHALLENGE).payload)), fixtureFile(STATEFUL, "presence-warming.bin")); signer.reportHealth("active"); const active = signer.answerRouteObservation(CHALLENGE); - assert.deepEqual(Buffer.from(active.payload), fixtureFile(STATEFUL, "presence-active.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(active.payload)), fixtureFile(STATEFUL, "presence-active.bin")); assert.match(String(presenceSlot(active, RUNTIME_PRESENCE_SLOT.writeLeaseSha256)), /^sha256-[0-9a-f]{64}$/); const degraded = statefulSigner(context); degraded.answerRouteObservation(CHALLENGE); degraded.reportHealth("degraded"); - assert.deepEqual(Buffer.from(degraded.answerRouteObservation(CHALLENGE).payload), fixtureFile(STATEFUL, "presence-degraded.bin")); + assert.deepEqual(Buffer.from(documentPayloadBytes(degraded.answerRouteObservation(CHALLENGE).payload)), fixtureFile(STATEFUL, "presence-degraded.bin")); }); test("a warming answered on the route becomes a lease target: Rust's lease naming it is accepted", (context) => { @@ -321,7 +322,7 @@ test("a warming answered on the route becomes a lease target: Rust's lease namin const signer = statefulSigner(context); assert.throws(() => signer.assertWriteLease(), /does not match the current Expected incarnation/, "no warming signed yet"); signer.answerRouteObservation(CHALLENGE); - assert.equal(signer.assertWriteLease(), presenceSlot({ payload: fixtureFile(STATEFUL, "presence-active.bin") }, RUNTIME_PRESENCE_SLOT.writeLeaseSha256)); + assert.equal(signer.assertWriteLease(), presenceSlot({ payload: documentPayloadFromBytes(fixtureFile(STATEFUL, "presence-active.bin")) }, RUNTIME_PRESENCE_SLOT.writeLeaseSha256)); }); for (const [lease, reason] of [["lease-other-warming.cc", "names another warming digest"], ["lease-other-target.cc", "names another target"]] as const) { diff --git a/packages/cultnet-ts/test/replication-codec.test.ts b/packages/cultnet-ts/test/replication-codec.test.ts index 0c6cd632..a93622ed 100644 --- a/packages/cultnet-ts/test/replication-codec.test.ts +++ b/packages/cultnet-ts/test/replication-codec.test.ts @@ -5,7 +5,7 @@ import { join, resolve } from "node:path"; import test from "node:test"; import { z } from "zod"; -import { CultCache, codecs, defineDocumentType, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; +import { CultCache, codecs, defineDocumentType, documentPayloadBytes, SingleFileMessagePackBackingStore } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, defineCultNetDocumentBinding } from "../src"; @@ -55,23 +55,23 @@ test("a codec document replicates as the bytes the C# reference writes, and a sn const target = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "target.cc"))).build(); const put = documents.createRawDocumentPutMessage(documents.get("tests.mixed_record")!, "put-1", "n1", value); - assert.equal(hex(put.document.payload), hex(reference), "a raw put carries the reference's record, not a map"); + assert.equal(hex(documentPayloadBytes(put.document.payload)), hex(reference), "a raw put carries the reference's record, not a map"); await documents.applyRawDocumentPutMessage(origin, put); - assert.equal(hex(origin.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(origin.getRequiredEnvelope(mixedDefinition, "n1").payload)), hex(reference)); const raw = documents.createRawSnapshotResponse(origin, "snapshot-raw"); - assert.equal(hex(raw.documents[0].payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(raw.documents[0].payload)), hex(reference)); await documents.applyRawSnapshotResponse(target, raw); assert.deepEqual(target.getRequired(mixedDefinition, "n1"), value); - assert.equal(hex(target.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(target.getRequiredEnvelope(mixedDefinition, "n1").payload)), hex(reference)); // The value-level snapshot decodes the stored record through the codec, so a peer applying it parses a record. const values = documents.createSnapshotResponse(origin, "snapshot-values"); assert.deepEqual(values.documents[0].payload, value); const other = CultCache.builder().withDocumentType(mixedDefinition).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "other.cc"))).build(); await documents.applySnapshotResponse(other, values); - assert.equal(hex(other.getRequiredEnvelope(mixedDefinition, "n1").payload), hex(reference)); + assert.equal(hex(documentPayloadBytes(other.getRequiredEnvelope(mixedDefinition, "n1").payload)), hex(reference)); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -100,5 +100,5 @@ test("a document with neither a codec nor a formatter replicates as a map, as be }); const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: plain })]); const put = documents.createRawDocumentPutMessage(documents.get("tests.plain_record")!, "put-3", "n1", { id: "n1" }); - assert.equal(hex(put.document.payload), "81a26964a26e31"); + assert.equal(hex(documentPayloadBytes(put.document.payload)), "81a26964a26e31"); }); diff --git a/scripts/check-typescript-tests-wired.mjs b/scripts/check-typescript-tests-wired.mjs new file mode 100644 index 00000000..8395699d --- /dev/null +++ b/scripts/check-typescript-tests-wired.mjs @@ -0,0 +1,54 @@ +// Fails when a TypeScript test file exists that no npm script of its package runs. +// +// A package's `test` script names the compiled files it runs, either by glob or one by one, and a file that is compiled +// and listed nowhere passes every suite while proving nothing: cultcache-ts's decode-document-payload test sat in that +// state behind an explicit two-file list. Every test/**/*.test.ts maps to its dist-test/**/*.test.js, and that path has +// to match an argument of `node --test` in the package's `test` script, or in `test:interop` for test/interop. +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const rootManifest = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")); + +function testFiles(dir, prefix = "") { + return readdirSync(dir, { withFileTypes: true }).flatMap(entry => + entry.isDirectory() + ? testFiles(join(dir, entry.name), `${prefix}${entry.name}/`) + : entry.name.endsWith(".test.ts") ? [`${prefix}${entry.name}`] : []); +} + +// The paths `node --test` is given in a script: the words of its command after `--test` that are not flags. +function nodeTestArguments(script) { + const words = script?.split("&&").map(part => part.trim().split(/\s+/)).find(part => part[0] === "node" && part.includes("--test")); + return words ? words.slice(words.indexOf("--test") + 1).filter(argument => !argument.startsWith("--")) : []; +} + +function matches(pattern, path) { + const [directory, file] = [dirname(pattern), pattern.slice(dirname(pattern).length + 1)]; + if (directory !== dirname(path)) return false; + return new RegExp(`^${file.split("*").map(part => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*")}$`) + .test(path.slice(directory.length + 1)); +} + +const unwired = []; +let checked = 0; +for (const workspace of rootManifest.workspaces) { + const packageRoot = join(repoRoot, workspace); + const scripts = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")).scripts ?? {}; + for (const file of testFiles(join(packageRoot, "test"))) { + checked += 1; + const compiled = `dist-test/test/${file.replace(/\.ts$/, ".js")}`; + const scriptName = file.startsWith("interop/") ? "test:interop" : "test"; + if (!nodeTestArguments(scripts[scriptName]).some(pattern => matches(pattern, compiled))) { + unwired.push(`${workspace}: test/${file} is not run by its package's ${scriptName} script`); + } + } +} + +if (checked === 0) throw new Error("The wiring check found no test files; it is looking in the wrong place."); +if (unwired.length > 0) { + console.error(unwired.join("\n")); + process.exit(1); +} +console.log(`${checked} TypeScript test files, all run by their package's script.`); diff --git a/scripts/run-typescript-workspaces.mjs b/scripts/run-typescript-workspaces.mjs index da881464..0ef34005 100644 --- a/scripts/run-typescript-workspaces.mjs +++ b/scripts/run-typescript-workspaces.mjs @@ -1,5 +1,6 @@ import { execFileSync } from "node:child_process"; import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; const command = process.argv[2]; if (!command) throw new Error("Usage: node scripts/run-typescript-workspaces.mjs