From be968a71e614202f538d4efabec0db20df983018 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:08:24 +0000 Subject: [PATCH 01/15] Bytes after the value: a payload is one MessagePack value (TypeScript) decodeDocumentPayload refuses bytes after the first value by count, from the checkPayloadDepth walk's end, with the shared clause; readWire's own trailing check is deleted. Walks bytes-after-outcomes.msgpack on the plain, codec, store and putEnvelope paths. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 2 + packages/cultcache-ts/README.md | 6 +- packages/cultcache-ts/src/document.ts | 8 +- packages/cultcache-ts/src/payload-depth.ts | 28 +++--- packages/cultcache-ts/src/wire.ts | 5 +- .../cultcache-ts/test/payload-depth.test.ts | 10 +++ .../cultcache-ts/test/union-parity.test.ts | 89 ++++++++++++++++++- 7 files changed, 128 insertions(+), 20 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index e71821c8..e45dc14e 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,9 +20,11 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. +- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication), refuses bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse, and a user formatter's bytes are not walked. The refused set is unchanged, but the message was `@msgpack`'s `RangeError` ("Extra N of M byte(s)") for a plain document and the codec reader's "bytes after its value" for a codec document, so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Implied bump: 0.16.0 (pre-1.0 minor). ### Added +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (it was `void`); it still refuses only depth. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 2096d258..041e90ae 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -49,7 +49,9 @@ through `decodeDocumentPayload`, and when you need the bytes themselves (hashing write) take them with `documentPayloadBytes`, the sanctioned escape. - `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter, - codec or plain map and then its schema. The definition is required. + codec or plain map and then its schema. The definition is required. A payload is exactly one + MessagePack value: bytes after the first are refused by count, naming the type, and never walked or + echoed (a payload that ends early is the decoder's to refuse). - `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter, codec or plain map. - `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a @@ -327,7 +329,7 @@ If you do not supply one, `CultCacheTS` uses MessagePack plus Zod validation by The 256-container nesting bound (`MAX_PAYLOAD_DEPTH`, refused with `CultCachePayloadDepthError`) covers the MessagePack the library itself writes: the default format and a codec. A payload your formatter writes is opaque bytes in any format, so the -library does not walk it as MessagePack; its format and its bounds are your formatter's. +library does not walk it as MessagePack; its format, its bounds and whether it is one value are your formatter's. Zod is no longer mandatory. Any schema object with a `parse(...)` function will do, which means generated Ajv-backed JSON-schema contracts can feed the cache diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 10af96a6..3443451e 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -119,7 +119,13 @@ export function decodeDocumentPayload { const bytes = documentPayloadBytes(payload); - if (writesLibraryMessagePack(definition)) checkPayloadDepth(bytes, `document type "${definition.type}" payload`); + if (writesLibraryMessagePack(definition)) { + // A payload is exactly one MessagePack value: bytes after the first are refused by count, from the walk's end, before + // any formatter reads; the tail is never walked or echoed. A payload that ends early stays the decoder's to refuse. + const subject = `document type "${definition.type}" payload`; + const end = checkPayloadDepth(bytes, subject); + if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + } const decoded = resolveDocumentFormatter(definition).decode(bytes); return definition.schema.parse(decoded) as CultCacheDocumentValue; } diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts index 40ca7afe..efbc3a74 100644 --- a/packages/cultcache-ts/src/payload-depth.ts +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -12,8 +12,9 @@ export class CultCachePayloadDepthError extends Error { // Walks the headers of one MessagePack value iteratively (a count stack bounded by the limit, scalars skipped by length, // nothing read) and throws at the first container past the limit. The message names the subject, the depth and the limit, -// never payload bytes. A payload that ends early is the reader's to refuse. -export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): void { +// never payload bytes. Returns the offset where the first value ends, or undefined when the payload ends early (the +// reader's to refuse); what follows that offset is never walked, and whether it may exist is the caller's rule. +export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): number | undefined { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const remaining = new Float64Array(MAX_PAYLOAD_DEPTH); let depth = 0; @@ -33,12 +34,12 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code <= 0x8f) { container = true; header = 1; children = 2 * (code & 0x0f); } else if (code <= 0x9f) { container = true; header = 1; children = code & 0x0f; } else if (code <= 0xbf) skip = 1 + (code & 0x1f); - else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return; skip = 2 + n; } - else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return; skip = 5 + n; } - else if (code === 0xc7) { const n = width(1); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc8) { const n = width(2); if (n === undefined) return; skip = 4 + n; } - else if (code === 0xc9) { const n = width(4); if (n === undefined) return; skip = 6 + n; } + else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return undefined; skip = 2 + n; } + else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return undefined; skip = 5 + n; } + else if (code === 0xc7) { const n = width(1); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc8) { const n = width(2); if (n === undefined) return undefined; skip = 4 + n; } + else if (code === 0xc9) { const n = width(4); if (n === undefined) return undefined; skip = 6 + n; } else if (code === 0xca || code === 0xce || code === 0xd2) skip = 5; else if (code === 0xcb || code === 0xcf || code === 0xd3) skip = 9; else if (code === 0xcc || code === 0xd0) skip = 2; @@ -47,10 +48,10 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code === 0xd6) skip = 6; else if (code === 0xd7) skip = 10; else if (code === 0xd8) skip = 18; - else if (code === 0xdc) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = n; } - else if (code === 0xdd) { const n = width(4); if (n === undefined) return; container = true; header = 5; children = n; } - else if (code === 0xde) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = 2 * n; } - else { const n = width(4); if (n === undefined) return; container = true; header = 5; children = 2 * n; } + else if (code === 0xdc) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = n; } + else if (code === 0xdd) { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = n; } + else if (code === 0xde) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = 2 * n; } + else { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = 2 * n; } if (container) { if (depth === MAX_PAYLOAD_DEPTH) throw new CultCachePayloadDepthError(MAX_PAYLOAD_DEPTH + 1, MAX_PAYLOAD_DEPTH, subject); @@ -65,6 +66,7 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload // One element is complete; close every container it was the last element of. while (depth > 0 && --remaining[depth - 1] === 0) depth--; - if (depth === 0) return; + if (depth === 0) return position; } + return undefined; } diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index 12bf572a..d351c446 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -128,7 +128,6 @@ export function readWire(payload: Uint8Array): unknown { } } - const tree = next(); - if (offset !== payload.length) throw new Error("the MessagePack payload has bytes after its value."); - return tree; + // One value is read; the caller owns what follows it (decodeDocumentPayload refuses a payload with a tail). + return next(); } diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index 382f0672..e792accb 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -119,6 +119,16 @@ test("a payload that ends early, is empty or carries trailing bytes is the reade assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0))); }); +test("the walk returns where the first value ends, and undefined when the payload ends early", () => { + assert.equal(checkPayloadDepth(bytes(0xc0)), 1); + assert.equal(checkPayloadDepth(bytes(0xc0, 0xc1, 0xc1)), 1, "a top-level scalar ends at its own length; the tail is not judged"); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01, 0xa1, 0x61, 0xc0)), 5 - 1, "a container ends when its last child completes"); + assert.equal(checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), 1, "the tail is never walked"); + assert.equal(checkPayloadDepth(bytes()), undefined); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined); + assert.equal(checkPayloadDepth(bytes(0xd9)), undefined); +}); + test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { refused(arrays(100_000), "100000 nested arrays"); refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps"); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 46dcb42f..f71924cc 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -515,7 +515,7 @@ test("the wire reader keeps 64-bit integers whole, tells a float from an integer assert.throws(() => readWire(fromHex("dd")), /truncated/u); assert.throws(() => readWire(fromHex("ddffffffff")), /truncated/u, "a huge array header reserves nothing"); assert.throws(() => readWire(fromHex("a5616263")), /truncated/u); - assert.throws(() => readWire(fromHex("0102")), /bytes after its value/u); + assert.equal(readWire(fromHex("0102")), 1, "the reader reads one value; the document owner refuses what follows"); assert.throws(() => readWire(fromHex("81a161a162")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("c1")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("d4010a")), /map, extension or reserved/u); @@ -915,3 +915,90 @@ test("a cache load refuses a store whose record is past the limit, naming the ty const refusing = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(await read("depth-past-limit.msgpack"))])).build(); await assert.rejects(() => refusing.pullAllBackingStores(), (error: Error) => error instanceof CultCachePayloadDepthError && !error.message.includes(CANARY)); }); + +// ---- bytes-after vectors: a payload is exactly one MessagePack value (tests/vectors/union-parity/README.md) ---- + +// The C# reference writes the vectors: union-surface.msgpack followed by a tail the first value does not own. The one +// owner refuses by count, naming the type, from the walk's end; it never walks or echoes the tail. +async function bytesAfterEntries() { + const raw = decode(await read("bytes-after-outcomes.msgpack")) as unknown[][]; + return raw.map(([file, outcome, unionName, key, detail]) => ({ file: file as string, outcome: outcome as string, unionName: unionName as string, key: key as number | null, detail: detail as string })); +} + +// The C# vector writer's canary (BytesAfterVectorTests): it sits in the deep tail, so an echo of the tail would carry it. +const TAIL_CANARY = "CANARY-bytes-after-7d1e"; +const valueLength = async () => (await read("union-surface.msgpack")).length; +const plainDocument = () => defineDocumentType({ type: "tests.plain_surface", schema: z.unknown() }); +const codecDocument = () => defineDocumentType({ type: "tests.codec_surface", schema: surfaceSchema, codec: surfaceCodec }); + +async function assertBytesAfterRefused(file: string, tail: number, run: () => unknown | Promise, type: string) { + const clause = `${type}`; + await assert.rejects( + async () => run(), + (error: Error) => { + assert.ok(!(error instanceof CultCachePayloadDepthError), `${file}: the tail was walked`); + assert.ok(!error.message.includes(CANARY) && !error.message.includes(TAIL_CANARY), `${file}: the refusal echoed the tail`); + assert.ok(!error.message.includes("the limit is"), `${file}: the depth refusal`); + assert.ok(error.message.includes(clause), `${file}: names the type: ${error.message}`); + assert.ok(error.message.includes(`${tail} bytes after its first MessagePack value`), `${file}: names the count: ${error.message}`); + return true; + }, + file, + ); +} + +test("every bytes-after vector is refused by the one owner, by count, on the codec, plain and store paths", async () => { + const entries = await bytesAfterEntries(); + assert.equal(entries.length, 4); + const length = await valueLength(); + const surface = surfaceDocument(); + const plain = plainDocument(); + const codecOnly = codecDocument(); + const dir = await mkdtemp(join(tmpdir(), "bytes-after-")); + for (const { file, outcome, unionName, detail } of entries) { + assert.equal(outcome, "refused"); + assert.equal(unionName, "UnionSurface"); + assert.equal(detail, "bytes-after"); + const bytes = await read(file); + const tail = bytes.length - length; + assert.ok(tail > 0, file); + if (file === "bytes-after-deep-tail.msgpack") assert.ok(Buffer.from(bytes).includes(TAIL_CANARY), "the deep tail carries the canary the assertions look for"); + const payload = documentPayloadFromBytes(bytes); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(surface, payload), surface.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(plain, payload), plain.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(codecOnly, payload), codecOnly.type); + + // The store path: a single-file store holding the vector as a record payload refuses on load. + const path = join(dir, `${file}.cc`); + await new SingleFileMessagePackBackingStore(path).push(envelopeOf(bytes)); + const cache = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type); + + // The put path: an envelope arriving with the tail is refused and held nowhere. + const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([])).build(); + await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type); + } +}); + +test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => { + const whole = await read("union-surface.msgpack"); + const surface = surfaceDocument(); + for (const cut of [0, 1, 10, whole.length - 1]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(whole.slice(0, cut))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `truncated at ${cut}`, + ); + } + // A header cut short of its length is an early end too, whatever width it declares. + for (const early of ["dc00", "dd000000", "de00", "c4", "d9", "a5616263", "9291"]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(fromHex(early))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `early end ${early}`, + ); + } + const opaque = defineDocumentType({ type: "tests.opaque", schema: z.unknown(), formatter: { encode: (value: Uint8Array) => value, decode: (bytes: Uint8Array) => bytes } }); + const tailed = await read("bytes-after-nil.msgpack"); + assert.equal((decodeDocumentPayload(opaque, documentPayloadFromBytes(tailed)) as Uint8Array).length, tailed.length); +}); From 50a8ead871266672404073961d3eab7f0d6294da Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:34:35 +0000 Subject: [PATCH 02/15] Walk end is undefined for any early end; the inspector speaks the one clause; changelog files the entry under Changed Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 4 ++-- .../cultcache-ts/src/cult-cache-inspector.ts | 18 ++++++++++++++---- packages/cultcache-ts/src/document.ts | 9 +++------ packages/cultcache-ts/src/payload-depth.ts | 11 +++++++++++ .../cultcache-ts/test/payload-depth.test.ts | 10 ++++++++++ 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index e45dc14e..c43d17bc 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,15 +20,15 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. -- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication), refuses bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse, and a user formatter's bytes are not walked. The refused set is unchanged, but the message was `@msgpack`'s `RangeError` ("Extra N of M byte(s)") for a plain document and the codec reader's "bytes after its value" for a codec document, so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Implied bump: 0.16.0 (pre-1.0 minor). ### Added -- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (it was `void`); it still refuses only depth. +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. `checkPayloadIsOneValue(bytes, subject)` is the depth walk plus the bytes-after refusal. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed +- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication, the provider command), and the inspector refuse bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse. The refused set is unchanged; only the text changed (it was `@msgpack`'s `RangeError` "Extra N of M byte(s)" for a plain document and the codec reader's "bytes after its value" for a codec document, and the inspector's preview no longer shows a tail), so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Stated limit: a definition with its own `formatter` is not walked, so whether its payload is one value is the formatter's. Implied bump: 0.16.0 (pre-1.0 minor, with the Breaking entries above). - Opening a legacy (pre-v1) store is a pure read: payloads that were not bytes are normalized in memory and the file is left as it was, so a codec that refuses the content cannot lose the old bytes. The next push, delete or flush rewrites the file as v1, merging what the open read. diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index 1c359157..71643913 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -1,6 +1,6 @@ import { decode } from "@msgpack/msgpack"; -import { checkPayloadDepth } from "./payload-depth"; +import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth"; import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; export interface CultCacheInspection { @@ -235,12 +235,22 @@ function decodeLegacyEnvelope(value: unknown): InspectedRecord { }; } -// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it. +// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it, +// and bytes after the first value are refused with the shared clause (the tail is never decoded or shown). function decodeBoundedPayload(payload: Uint8Array): unknown { - checkPayloadDepth(payload); + checkPayloadIsOneValue(payload, "document payload"); return decode(payload); } +// Where the first value ends (the whole payload when it ends early or cannot be walked): a preview never shows a tail. +function valueEnd(payload: Uint8Array): number { + try { + return checkPayloadDepth(payload) ?? payload.length; + } catch { + return payload.length; + } +} + function previewPayload(payload: Uint8Array): Pick { try { return { payloadPreview: toJsonSafe(decodeBoundedPayload(payload)) }; @@ -248,7 +258,7 @@ function previewPayload(payload: Uint8Array): Pick value.toString(16).padStart(2, "0")) .join(" "), }, diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 3443451e..ac6e5f7d 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -1,5 +1,5 @@ import { resolveDocumentFormatter } from "./document-formatter"; -import { checkPayloadDepth } from "./payload-depth"; +import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth"; import type { AnyCultCacheDocumentDefinition, CultCacheSchemaCatalogMember, @@ -120,11 +120,8 @@ export function decodeDocumentPayload { const bytes = documentPayloadBytes(payload); if (writesLibraryMessagePack(definition)) { - // A payload is exactly one MessagePack value: bytes after the first are refused by count, from the walk's end, before - // any formatter reads; the tail is never walked or echoed. A payload that ends early stays the decoder's to refuse. - const subject = `document type "${definition.type}" payload`; - const end = checkPayloadDepth(bytes, subject); - if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + // A payload is exactly one MessagePack value: bytes after the first are refused by count before any formatter reads. + checkPayloadIsOneValue(bytes, `document type "${definition.type}" payload`); } const decoded = resolveDocumentFormatter(definition).decode(bytes); return definition.schema.parse(decoded) as CultCacheDocumentValue; diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts index efbc3a74..45b8d5ca 100644 --- a/packages/cultcache-ts/src/payload-depth.ts +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -64,9 +64,20 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload position += skip; } + // A scalar whose declared length runs past the payload is an early end, not a value. + if (position > bytes.length) return undefined; // One element is complete; close every container it was the last element of. while (depth > 0 && --remaining[depth - 1] === 0) depth--; if (depth === 0) return position; } return undefined; } + +// The one rule that a payload is exactly one MessagePack value: the depth walk, then a refusal of bytes after the first +// value, by count, naming the subject, from the walk's end. The tail is never walked or echoed; a payload that ends early +// stays the decoder's to refuse. decodeDocumentPayload and the inspector speak this clause and no other. +export function checkPayloadIsOneValue(bytes: Uint8Array, subject: string): number | undefined { + const end = checkPayloadDepth(bytes, subject); + if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + return end; +} diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index e792accb..6fb81265 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -127,6 +127,9 @@ test("the walk returns where the first value ends, and undefined when the payloa assert.equal(checkPayloadDepth(bytes()), undefined); assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined); assert.equal(checkPayloadDepth(bytes(0xd9)), undefined); + for (const cut of [[0xcc], [0xcb, 0x3f, 0xf8], [0xa5, 0x61, 0x62, 0x63], [0xc4, 0x02, 0x01], [0xd7, 0x01]]) { + assert.equal(checkPayloadDepth(bytes(...cut)), undefined, `a scalar cut short: ${cut.map((value) => value.toString(16)).join(" ")}`); + } }); test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { @@ -208,6 +211,13 @@ test("the inspector previews a payload at the limit and reports one past it, or } }); +test("the inspector refuses bytes after a payload's first value with the shared clause and previews no tail", async () => { + const file = await storeWith(bytes(0xc0, 0xc1, 0xc1)); + const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!; + assert.equal(record.payloadDecodeError, "document payload has 2 bytes after its first MessagePack value."); + assert.deepEqual(record.payloadPreview, { bytes: 3, hexPrefix: "c0" }); +}); + // A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not. const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff); From e6bd81adcb03cc80934e67c05d3b9998f0089d0c Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:34:35 +0000 Subject: [PATCH 03/15] Pin the tail refusal on CultNet replication, the provider command and the CDN manifest; held-nowhere is asserted Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-ts/test/union-parity.test.ts | 5 +++- .../test/provider-session-wire.test.ts | 13 +++++++++++ packages/cultnet-ts/test/content.test.ts | 7 ++++++ packages/cultnet-ts/test/cultnet.test.ts | 23 +++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index f71924cc..00554bc3 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -975,8 +975,11 @@ test("every bytes-after vector is refused by the one owner, by count, on the cod await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type); // The put path: an envelope arriving with the tail is refused and held nowhere. - const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([])).build(); + const written: CultCacheEnvelope[] = []; + const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build(); await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type); + assert.equal(written.length, 0, `${file}: nothing reached the store`); + assert.equal(held.get(surface, "surface"), undefined, `${file}: nothing is held in memory`); } }); diff --git a/packages/cultmesh-ts/test/provider-session-wire.test.ts b/packages/cultmesh-ts/test/provider-session-wire.test.ts index 4905f161..69314fbe 100644 --- a/packages/cultmesh-ts/test/provider-session-wire.test.ts +++ b/packages/cultmesh-ts/test/provider-session-wire.test.ts @@ -83,3 +83,16 @@ test("a provider command that is not a map, or lacks a required field, is refuse (error: Error) => error.message === "CultMesh provider command must be a MessagePack map.", ); }); + +test("a provider command payload with bytes after its first value is refused by count, without echoing the tail", () => { + const command = { commandId: "c1", commandKind: "k", providerId: "p", serviceInstanceId: "s", payload: { n: 1 } }; + const whole = documentPayloadBytes(encodeDocumentPayload(providerCommandDocument, command)); + const tailed = documentPayloadFromBytes(Uint8Array.from([...whole, ...Buffer.from("canary-tail-8c1f")])); + assert.throws( + () => decodeProviderCommandDocument({ + schemaId: cultMeshProviderSessionSchemas.command, recordKey: "p/s/c1", storedAt: "2026-10-11T00:00:00Z", + payloadEncoding: "messagepack" as const, payload: tailed, + }), + (error: Error) => error.message.includes("16 bytes after its first MessagePack value") && !error.message.includes("canary-tail-8c1f"), + ); +}); diff --git a/packages/cultnet-ts/test/content.test.ts b/packages/cultnet-ts/test/content.test.ts index ec23b64e..e2ae5ed9 100644 --- a/packages/cultnet-ts/test/content.test.ts +++ b/packages/cultnet-ts/test/content.test.ts @@ -392,6 +392,13 @@ test("decodeManifest accepts and refuses exactly what the reference does, and re } }); +test("a manifest with bytes after its first value is refused by count, whatever the tail is", () => { + const whole = encodeManifest(manifestOf()); + for (const tail of [[0xc0], [0xc1, 0xc1], [0x90]]) { + assert.throws(() => decodeManifest(Uint8Array.from([...whole, ...tail])), new RegExp(`${tail.length} bytes after its first MessagePack value`)); + } +}); + test("a manifest written as a map names the manifest, not a field inside it", () => { assert.throws(() => decodeManifest(encode({ 0: "a", 9: { k: "v" } })), /field 'manifest' has the wrong type/); }); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index ac4f539d..581fd66a 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -14,6 +14,8 @@ import { SchemaConflictError, SingleFileMessagePackBackingStore, defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, schemaIdentityOf, } from "@gamecult/cultcache-ts"; @@ -3140,3 +3142,24 @@ test("CultNet registry names the canonical schema id of a compatible alias, and assert.equal(registry.canonicalSchemaId("p3.nobody"), undefined); assert.equal(registry.canonicalSchemaId("p3.shared"), undefined, "an id several bindings claim resolves to none"); }); + +test("a replicated record with bytes after its first MessagePack value is refused by count and held nowhere", async () => { + const definition = defineDocumentType({ type: "tails.note", schemaId: "tails.note.v1", schema: z.object({ body: z.string() }) }); + const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition })]); + const origin = CultCache.builder().withDocumentType(definition).build(); + await origin.put(definition, "k", { body: "x" }); + const document = registry.createRawSnapshotResponse(origin, "tails").documents[0]!; + const whole = documentPayloadBytes(document.payload); + const target = CultCache.builder().withDocumentType(definition).build(); + await assert.rejects( + registry.applyRawDocumentPutMessage(target, { + schemaVersion: "cultnet.document_put_raw.v0", + messageId: "tails", + document: { ...document, payload: documentPayloadFromBytes(Uint8Array.from([...whole, 0xc0, 0xc1])) }, + }), + (error: Error) => error.message.includes('document type "tails.note" payload has 2 bytes after its first MessagePack value'), + ); + assert.deepEqual(target.snapshot(), []); + await registry.applyRawDocumentPutMessage(target, { schemaVersion: "cultnet.document_put_raw.v0", messageId: "whole", document }); + assert.deepEqual(target.getRequired(definition, "k"), { body: "x" }); +}); From 8c36ce4771b7467dc9789949fcc06a00a7b3a052 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:41:47 +0000 Subject: [PATCH 04/15] Changelog: the one-value check is internal, not new API Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-ts/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index c43d17bc..9f0301cc 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -23,7 +23,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Added -- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. `checkPayloadIsOneValue(bytes, subject)` is the depth walk plus the bytes-after refusal. +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. The bytes-after refusal is internal (`checkPayloadIsOneValue`, not exported); no new public name. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed From 34d85b2b597e316265faa208a3ffbad6178ae4b2 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 05:06:33 +0000 Subject: [PATCH 05/15] Pin: a tail on a payload with unread slots is refused before any slot is marked or the record held Co-Authored-By: Claude Sonnet 5.5 --- .../cultcache-ts/test/union-parity.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 7c6534b4..d917ea7a 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -793,6 +793,27 @@ test("every bytes-after vector is refused by the one owner, by count, on the cod } }); +test("a payload with an unread slot and a tail is refused by count before any slot is marked or the record held", async () => { + const withSlot = await read("extra-slot-record.msgpack"); + const surface = surfaceDocument(); + const tailed = Uint8Array.from([...withSlot, 0xc0]); + const whole = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(withSlot)])).build(); + await whole.pullAllBackingStores(); + assert.ok(whole.unreadSlots(surface, "surface"), "without a tail the record is held with its unread slots"); + + const loading = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(tailed)])).build(); + await assertBytesAfterRefused("extra-slot-record+tail", 1, () => loading.pullAllBackingStores(), surface.type); + assert.equal(loading.get(surface, "surface"), undefined); + assert.equal(loading.unreadSlots(surface, "surface"), undefined); + + const written: CultCacheEnvelope[] = []; + const putting = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build(); + await assertBytesAfterRefused("extra-slot-record+tail", 1, () => putting.putEnvelope(surface, envelopeOf(tailed)), surface.type); + assert.equal(written.length, 0); + assert.equal(putting.get(surface, "surface"), undefined); + assert.equal(putting.unreadSlots(surface, "surface"), undefined); +}); + test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => { const whole = await read("union-surface.msgpack"); const surface = surfaceDocument(); From 13ce5ea50a0e231819eb6f33ae76db1296ac06d2 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 05:23:25 +0000 Subject: [PATCH 06/15] Python: a payload is one MessagePack value (bytes-after-py) (#129) --- packages/cultcache-py/CHANGELOG.md | 6 +- packages/cultcache-py/README.md | 1 + packages/cultcache-py/pyproject.toml | 2 +- .../cultcache-py/src/cultcache_py/__init__.py | 2 + .../src/cultcache_py/documents.py | 26 ++++- .../cultcache-py/tests/test_union_parity.py | 72 ++++++++++++++ packages/cultmesh-py/pyproject.toml | 2 +- packages/cultnet-py/CHANGELOG.md | 1 + packages/cultnet-py/pyproject.toml | 2 +- .../cultnet-py/src/cultnet_py/messages.py | 3 +- .../cultnet-py/src/cultnet_py/replication.py | 6 +- packages/cultnet-py/tests/test_cultnet.py | 45 +++++++++ .../tests/test_declared_dependency_ranges.py | 94 +++++++++++++++++++ 13 files changed, 251 insertions(+), 11 deletions(-) create mode 100644 packages/cultnet-py/tests/test_declared_dependency_ranges.py diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index 08fb4d1f..ea3f72fd 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Added + +- `read_payload_tree(type_name, payload, decoder=None)`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged. + ### Changed - `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import. @@ -10,7 +14,7 @@ ### Breaking - A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`. - +- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` ` payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#. - A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it. - Added `foreign_records()` and the frozen `ForeignRecord(key, schema_id, schema_name, stored_at)`, sorted by key in UTF-16 code-unit order. `stores.retyped` and `stores.written_owns` lose their underscore: the cache judge and the single-file store's commit guard ask one predicate. diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 72d51562..4a7236fd 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -19,6 +19,7 @@ discovery, sessions, local server) build on it, mirroring `cultcache-rs` / `[format, schemaCatalog, records]` - records store raw MessagePack payload bytes under schema-catalog identity - payloads decode only through the registered document definition for that type +- a payload is exactly one MessagePack value: bytes after it are refused on read, by count - a cache holds one record per key whatever its type: a write of a second type at a held key is refused with `SchemaConflictError` (in memory too), and two stores holding one key refuse the load - a store record no registered type claims loads and is carried as foreign: `cache.foreign_records()` lists it, the store diff --git a/packages/cultcache-py/pyproject.toml b/packages/cultcache-py/pyproject.toml index 6c46d102..15b724ba 100644 --- a/packages/cultcache-py/pyproject.toml +++ b/packages/cultcache-py/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cultcache-py" -version = "0.3.0" +version = "0.4.0" description = "Python CultCache runtime for GameCult typed state." readme = "README.md" requires-python = ">=3.10" diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index 4873b6ed..69828bf3 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -16,6 +16,7 @@ from .documents import ( define_document_type, define_record_codec, define_union, + read_payload_tree, ) from .leaves import SlotCodec from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError @@ -40,6 +41,7 @@ __all__ = [ "StoreUnreadableError", "UnionArm", "UnreadSlots", + "read_payload_tree", "leaves", "database_entry_field", "define_database_entry_type", diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 4f7dfe7d..ff9039dc 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -158,6 +158,26 @@ def _messagepack_payload_decoder(payload: bytes) -> Any: return msgpack.unpackb(payload, raw=False) +def read_payload_tree(type_name: str, payload: bytes, decoder: Callable[[bytes], Any] | None = None) -> Any: + """The one read of a payload's tree: exactly one MessagePack value, within the nesting limit. Depth is judged first, + on the value read so far, so a tail never pre-empts it (C# throws at container 257 before it reaches the tail); then a + tail is refused by count and never echoed. Raised outside the handler so no `__context__` keeps the tail.""" + after = -1 + unpacked: Any = None + try: + tree = (decoder or _messagepack_payload_decoder)(payload) + except _STACK_FAULTS: + raise CultCachePayloadDepthError(type_name, "read") from None + except msgpack.ExtraData as error: + after = len(error.extra) + unpacked = error.unpacked + if after >= 0: + _refuse_past_depth_limit(unpacked, type_name, "read") + raise ValueError(f"{type_name} payload has {after} bytes after its first MessagePack value (refused on read).") + _refuse_past_depth_limit(tree, type_name, "read") + return tree + + @dataclass(frozen=True) class DatabaseEntryField: name: str @@ -234,11 +254,7 @@ class DocumentDefinition(Generic[T]): def decode_payload_marked(self, payload: bytes) -> tuple[T, UnreadSlots | None]: """The value and what the decode found past the declared slots, at any depth. The payload depth check runs first: a payload past the limit is refused, never marked.""" - try: - tree = self.payload_decoder(payload) - except _STACK_FAULTS: - raise CultCachePayloadDepthError(self.type, "read") from None - _refuse_past_depth_limit(tree, self.type, "read") + tree = read_payload_tree(self.type, payload, self.payload_decoder) outer, opened = _current_scope(), _SlotScope() _scope.open = opened try: diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index 1e19ce8d..793933ff 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -661,6 +661,78 @@ def _as_tuple(tree: object) -> object: return tuple(_as_tuple(child) for child in tree) if isinstance(tree, list) else tree + +class BytesAfterVectorTests(unittest.TestCase): + CLAUSE = "bytes after its first MessagePack value (refused on read)" + + def _assert_chain_clean(self, error: BaseException, file: str) -> None: + link: BaseException | None = error + while link is not None: + self.assertNotIn(CANARY, str(link), file) + self.assertNotIsInstance(link, msgpack.ExtraData, file) + link = link.__cause__ or link.__context__ + + def test_every_bytes_after_vector_is_refused(self) -> None: + base = _vector("union-surface.msgpack") + entries = msgpack.unpackb(_vector("bytes-after-outcomes.msgpack"), raw=False) + self.assertEqual(len(entries), 4) + for file, outcome_kind, union, key, detail in entries: + self.assertEqual((outcome_kind, union, key, detail), ("refused", "UnionSurface", None, "bytes-after"), file) + payload = _vector(file) + self.assertTrue(payload.startswith(base), file) + count = len(payload) - len(base) + with self.assertRaises(ValueError, msg=file) as caught: + surface_document.decode_payload(payload) + error = caught.exception + self.assertIs(type(error), ValueError, file) + self.assertNotIsInstance(error, CultCachePayloadDepthError, file) + self.assertEqual( + str(error), f"{surface_document.type} payload has {count} {self.CLAUSE}.", file + ) + self.assertNotIn("the limit is", str(error), file) + self._assert_chain_clean(error, file) + + def test_a_single_file_store_holding_a_tail_as_a_record_payload_refuses_on_load(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "tail.cc" + shutil.copyfile(VECTORS / "union-surface.cc", path) + store = SingleFileMessagePackBackingStore(path) + [envelope] = store.pull_all() + store.push(dataclasses.replace(envelope, payload=_vector("bytes-after-nil.msgpack"))) + cache = ( + CultCache.builder() + .register_document_type(surface_document) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + with self.assertRaises(ValueError) as caught: + cache.pull_all_backing_stores() + self.assertIn(f"1 {self.CLAUSE}", str(caught.exception)) + self.assertIn(surface_document.type, str(caught.exception)) + self.assertIsNone(cache.get(surface_document, "surface")) + + def test_depth_wins_over_a_tail_whatever_the_unpackers_own_stack_limit(self) -> None: + # Vector-independent: a value nested past 256 is the depth refusal even with a tail, as C# throws at container 257 + # before it reaches the tail; 2000 is past msgpack's own stack, so the winner is not a library constant. + for depth in (257, 300, 900, 2000, 100000): + for tail in (b"\xc0", b"\xc1\xc1"): + with self.subTest(depth=depth, tail=len(tail)): + with self.assertRaises(CultCachePayloadDepthError) as caught: + surface_document.decode_payload(b"\x91" * depth + b"\xc0" + tail) + self.assertIn("refused on read", str(caught.exception)) + for depth in (1, 255, 256): + with self.subTest(at=depth): + with self.assertRaises(ValueError) as caught: + surface_document.decode_payload(b"\x91" * depth + b"\xc0\xc0") + self.assertNotIsInstance(caught.exception, CultCachePayloadDepthError) + self.assertIn(f"1 {self.CLAUSE}", str(caught.exception)) + + def test_a_payload_that_ends_early_is_the_decoders_not_bytes_after(self) -> None: + truncated = _vector("union-surface.msgpack")[:-1] + with self.assertRaises(Exception) as caught: + surface_document.decode_payload(truncated) + self.assertNotIn("bytes after", str(caught.exception)) + class PayloadDepthRuleTests(unittest.TestCase): """The nesting rule at its seam, on definitions that are not unions: the same routine serves any payload codec.""" diff --git a/packages/cultmesh-py/pyproject.toml b/packages/cultmesh-py/pyproject.toml index 5f9bda4d..18f23060 100644 --- a/packages/cultmesh-py/pyproject.toml +++ b/packages/cultmesh-py/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0", "cryptography>=41"] +dependencies = ["cultcache-py>=0.4.0", "cultnet-py>=0.1.0", "cryptography>=41"] [project.scripts] cultmesh-py-daemon = "cultmesh_py.daemon:main" cultmesh-py-verify = "cultmesh_py.verify:main" diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index b8a3773e..525bd4bd 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). A raw record with bytes after its first value gives no schema hint to `resolve_document_for_raw_record`, so an unknown schema id stays `KeyError` rather than being routed; a registered one is refused by count when applied. - `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out. - `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move. - Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client. diff --git a/packages/cultnet-py/pyproject.toml b/packages/cultnet-py/pyproject.toml index 41c4e8f6..09db2733 100644 --- a/packages/cultnet-py/pyproject.toml +++ b/packages/cultnet-py/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["cultcache-py>=0.2.0", "msgpack>=1.0"] +dependencies = ["cultcache-py>=0.4.0", "msgpack>=1.0"] [project.optional-dependencies] crypto = ["cryptography>=42"] diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index d5e26b7a..c9b7d4a6 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Any import msgpack +from cultcache_py import read_payload_tree from .unicode_messageid_tables import DEFAULT_IGNORABLE, LETTER_OR_NUMBER @@ -544,7 +545,7 @@ def encode_witness_artifact_bundle_payload(bundle: dict[str, Any] | CultNetWitne def decode_witness_artifact_bundle_payload(payload: bytes) -> dict[str, Any]: - slots = msgpack.unpackb(payload, raw=False) + slots = read_payload_tree("cultnet.witness_artifact_bundle", payload) if not isinstance(slots, list) or len(slots) < 8: raise ValueError("CultNet witness artifact bundle payload must be an 8-slot MessagePack array") return { diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 531c0012..28f5da33 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -171,9 +171,13 @@ def resolve_document_for_raw_record( def _infer_schema_version_from_payload(payload: bytes) -> str | None: + import msgpack # type: ignore try: - import msgpack # type: ignore decoded = msgpack.unpackb(payload, raw=False) + except msgpack.ExtraData: + # A payload with bytes after its value gives no routing hint: the record is unroutable by inference, and a + # registered schema id routes it to a definition whose decode refuses the tail by count. + return None except Exception: return None if isinstance(decoded, list) and decoded and isinstance(decoded[0], str): diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 48972221..cdfaee5b 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -1971,6 +1971,34 @@ class CultNetTests(unittest.TestCase): apply_raw_document_record(cache, schema_document_map([first, second]), raw(first)) # the same type still replaces self.assertEqual(cache.get_required(first, "shared").name, "remote") + def test_a_raw_record_with_a_tail_is_unroutable_by_inference_and_refused_by_count_when_routed(self) -> None: + import msgpack # type: ignore + from cultcache_py import define_document_type + from cultnet_py import resolve_document_for_raw_record + + document = define_document_type("tests.tail_note", schema_id="tests.tail_note", schema_version="tests.tail_note.v1", + payload_encoder=msgpack.packb, payload_decoder=lambda payload: msgpack.unpackb(payload, raw=False)) + known = document.catalog_entry().schema_id + head = msgpack.packb(["tests.tail_note.v1", "x"], use_bin_type=True) + + def raw(schema_id: str, payload: bytes) -> dict: + return {"schemaId": schema_id, "recordKey": "k", "storedAt": "2026-06-13T00:00:00Z", + "payloadEncoding": "messagepack", "payload": payload} + + documents = schema_document_map([document]) + # Routing by inference reads a hint, not a payload: a payload with a tail gives no hint, so an unknown schema id + # stays the unknown-schema error (KeyError) even when the first value names a registered schema. + self.assertIs(resolve_document_for_raw_record(documents, "unknown", raw("unknown", head)), document) + with self.assertRaises(KeyError): + resolve_document_for_raw_record(documents, "unknown", raw("unknown", head + b"\xc0")) + # A registered schema id routes, and the tail is refused by count when the record is applied. + cache = CultCache() + cache.register_document_type(document) + with self.assertRaises(ValueError) as caught: + apply_raw_document_record(cache, documents, raw(known, head + b"\xc0")) + self.assertIn("1 bytes after its first MessagePack value", str(caught.exception)) + self.assertIsNone(cache.get(document, "k")) + def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None: document = define_database_entry_type( "replica.item", @@ -2661,6 +2689,23 @@ class CultNetTests(unittest.TestCase): self.assertEqual(responses[0]["details"]["value"], schema_version) self.assertNotIn("messageId", responses[0]) + def test_witness_artifact_bundle_payload_is_one_value_and_depth_wins_over_a_tail(self) -> None: + import msgpack # type: ignore + from cultcache_py import CultCachePayloadDepthError + + payload = msgpack.packb(["b", "k", "t", {}, [], [], [], {}], use_bin_type=True) + decode_witness_artifact_bundle_payload(payload) + for tail, count in ((b"\xc0", 1), (b"\xc1\xc1", 2)): + with self.assertRaises(ValueError) as caught: + decode_witness_artifact_bundle_payload(payload + tail) + self.assertIs(type(caught.exception), ValueError) + self.assertEqual( + str(caught.exception), + f"cultnet.witness_artifact_bundle payload has {count} bytes after its first MessagePack value (refused on read).", + ) + with self.assertRaises(CultCachePayloadDepthError): + decode_witness_artifact_bundle_payload(b"\x91" * 300 + b"\xc0\xc0") + def test_cultnet_simulation_consensus_dedupes_witnesses_and_requires_quorum(self) -> None: from cultnet_py import CultNetSimulationConsensus diff --git a/packages/cultnet-py/tests/test_declared_dependency_ranges.py b/packages/cultnet-py/tests/test_declared_dependency_ranges.py new file mode 100644 index 00000000..9baf15fc --- /dev/null +++ b/packages/cultnet-py/tests/test_declared_dependency_ranges.py @@ -0,0 +1,94 @@ +"""A sibling package resolves through the workspace's editable install, so a package that starts importing a new name from +cultcache-py passes whatever range it declares. A consumer outside the workspace installs the newest release the range +admits. This is the Python form of scripts/test-declared-dependency-ranges.mjs: for every tagged cultcache-py release +(`cultcache-py-vX.Y.Z`) that the `cultcache-py>=` bound of cultnet-py or cultmesh-py admits, every name those packages +import from cultcache_py must be defined by that release's source. It needs git with the release tags and skips without +them. Only `>=` bounds are read; star imports and names reached by attribute access are not checked.""" + +from __future__ import annotations + +import ast +import re +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +PACKAGES = ROOT / "packages" +CONSUMERS = ("cultnet-py", "cultmesh-py") + + +def _version(text: str) -> tuple[int, ...]: + return tuple(int(part) for part in text.split(".")) + + +def _git(*args: str) -> str | None: + try: + done = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True) + except (OSError, subprocess.CalledProcessError): + return None + return done.stdout + + +def _lower_bound(package: str) -> tuple[int, ...]: + manifest = (PACKAGES / package / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'"cultcache-py>=(\d+\.\d+\.\d+)"', manifest) + assert match, f"{package} declares no cultcache-py>= bound" + return _version(match.group(1)) + + +def _imports(package: str) -> dict[str, set[str]]: + """module -> names imported from it, for every `from cultcache_py[.x] import names` under the package's src.""" + found: dict[str, set[str]] = {} + for source in (PACKAGES / package / "src").rglob("*.py"): + for node in ast.walk(ast.parse(source.read_text(encoding="utf-8-sig"))): + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + if node.module == "cultcache_py" or node.module.startswith("cultcache_py."): + found.setdefault(node.module, set()).update(alias.name for alias in node.names if alias.name != "*") + return found + + +def _defined(tag: str, module: str) -> set[str] | None: + base = "packages/cultcache-py/src/" + module.replace(".", "/") + for path in (base + ".py", base + "/__init__.py"): + text = _git("show", f"{tag}:{path}") + if text is None: + continue + names: set[str] = set() + for node in ast.parse(text.lstrip("\ufeff")).body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update((alias.asname or alias.name).split(".")[0] for alias in node.names) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + names.add(target.id) + return names + return None + + +class DeclaredDependencyRangeTests(unittest.TestCase): + def test_every_admitted_cultcache_release_defines_what_the_consumers_import(self) -> None: + listed = _git("tag", "--list", "cultcache-py-v*") + tags = sorted((t for t in (listed or "").split() if re.fullmatch(r"cultcache-py-v\d+\.\d+\.\d+", t)), + key=lambda t: _version(t.rsplit("v", 1)[1])) + if not tags: + self.skipTest("git with the cultcache-py release tags is not available") + current = re.search(r'^version = "(\d+\.\d+\.\d+)"', (PACKAGES / "cultcache-py" / "pyproject.toml").read_text(encoding="utf-8"), re.M) + assert current + for package in CONSUMERS: + bound = _lower_bound(package) + self.assertLessEqual(bound, _version(current.group(1)), f"{package}'s bound is above the workspace cultcache-py") + imports = _imports(package) + for tag in tags: + if _version(tag.rsplit("v", 1)[1]) < bound: + continue + for module, names in imports.items(): + defined = _defined(tag, module) + self.assertIsNotNone(defined, f"{package} imports {module}, which {tag} does not have") + self.assertEqual(sorted(names - defined), [], f"{package} imports names {tag} lacks from {module}") + + +if __name__ == "__main__": + unittest.main() From 363a8d6fe5b0e09330fe5a2f26bcdb80238e5dd5 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 05:43:55 +0000 Subject: [PATCH 07/15] Rust: a payload is one MessagePack value (bytes-after-rs) (#131) --- docs/cultcache-union-parity-map.md | 2 +- packages/cultcache-rs/CHANGELOG.md | 2 + packages/cultcache-rs/src/lib.rs | 137 +++++++++++++--- packages/cultcache-rs/tests/slot_readonly.rs | 52 +++++- packages/cultcache-rs/tests/union_parity.rs | 162 ++++++++++++++++++- packages/cultnet-rs/CHANGELOG.md | 1 + packages/cultnet-rs/tests/cultnet.rs | 10 +- packages/cultnet-rs/tests/decode_door.rs | 10 ++ 8 files changed, 346 insertions(+), 30 deletions(-) diff --git a/docs/cultcache-union-parity-map.md b/docs/cultcache-union-parity-map.md index 368be16a..1442abdc 100644 --- a/docs/cultcache-union-parity-map.md +++ b/docs/cultcache-union-parity-map.md @@ -259,7 +259,7 @@ landed in `ead5260c`, Python in `5618d8dd`, Rust in `c028c511`. TypeScript is no | Runtime | Owner of the decision (constant and walker) | Doors routed through it | |---|---|---| | C# | `CultPayloadDepth.Check` (`src/GameCult.Caching.MessagePack/CultPayloadDepth.cs`), limit `CultMessagePackSecurity.MaximumPayloadDepth`. MessagePack's own graph counter is set to the same value as a backstop only. | Write: `CultDocumentMessagePackSerialization.SerializeUntyped`. Read: `DeserializeUntyped`. The cache store (`SerializePayload`/`DeserializePayload`), the directory store, `CultMesh` and single-file documents all call those two; C# has no separate raw-envelope or prepared-batch door. | -| Rust | `cultcache_rs::check_payload_depth` and `MAX_PAYLOAD_DEPTH` (`packages/cultcache-rs/src/lib.rs`). | Read: `decode_payload` (get, list, `validate_envelope`/`put_envelope`, `finish_prepared_entry`). Write: `put`/`prepare_entry`/`prepare_entry_named` reach it through `finish_prepared_entry`, which decodes what it just encoded. Raw envelope: `put_raw_envelope`, the door CultMesh network puts use (`cultmesh-rs` `lib.rs`, two call sites). Prepared batch: `put_prepared_batch` via `require_payload_depth`. cultnet-rs: every free-form or skippable wire payload decodes through `cultcache_rs::decode_payload` (message decode in `contracts.rs`, `provider_session`, `media_stream_wire`, `replication`, `service_identity`). | +| Rust | `check_record_payload` (private; depth, then no bytes after the first value) over `first_value_end`, and `MAX_PAYLOAD_DEPTH`; `cultcache_rs::check_payload_depth` is the depth-only probe for outside callers (`packages/cultcache-rs/src/lib.rs`). | Read: `decode_payload` (get, list, `validate_envelope`/`put_envelope`, `finish_prepared_entry`). Write: `put`/`prepare_entry`/`prepare_entry_named` reach it through `finish_prepared_entry`, which decodes what it just encoded. Raw envelope: `put_raw_envelope`, the door CultMesh network puts use (`cultmesh-rs` `lib.rs`, two call sites). Prepared batch: `put_prepared_batch` via `require_record_payload`. Load: `pull_all_backing_stores` and `load_soa` run `require_record_payload` on every record, so a store holding a payload past the bound (or with bytes after it) is refused at load, and the cache then refuses every write and removal (`CultCacheLoadRefused`) until a load succeeds. cultnet-rs: every free-form or skippable wire payload decodes through `cultcache_rs::decode_payload` (message decode in `contracts.rs`, `provider_session`, `media_stream_wire`, `replication`, `service_identity`). | | Python | `_refuse_past_depth_limit` and `MAX_PAYLOAD_DEPTH` (`packages/cultcache-py/src/cultcache_py/documents.py`); a msgpack `StackError` on read maps to the same error. | `DocumentDefinition.encode_payload` (write) and `decode_payload` (read). `CultCache.put`, `put_envelope`, `put_envelopes` and load-time hydration go through them; there is no raw-envelope door. cultmesh-py round trips call the same pair. | - **Doors deliberately not routed.** cultnet-rs `idunn_lifecycle_brake`, `provider_health_contracts` and diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index 353facea..4e97588c 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -15,6 +15,8 @@ Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0). - `SchemaConflictError` gains the public field `unread_slots: Option`; code that builds it with a struct literal must set it (`None`). - A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it. - The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check. +- A cache whose last `pull_all_backing_stores` was refused (a tail, a payload past the nesting bound, a duplicate unique-index value, a retype conflict, a record from the wrong store) refuses every `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch`, `update` and `delete` with `CultCacheLoadRefused` until a pull succeeds. Before, it stayed writable over an empty image, and the next batch write replaced the store with only the new records. +- `decode_payload`, `put_raw_envelope` and prepared batches refuse a payload with bytes after its first MessagePack value (`payload has {n} bytes after its first MessagePack value`), by count; the tail is never walked or quoted. They used to read the first value and keep or drop the tail, so a record with a stray tail was served and rewritten without it. A store holding such a record (single-file, redb, `load_soa`) is refused at load, naming the type, the key and the count, and stays unreadable until the record is repaired; the same check now runs on every record at load, as the C# and Python runtimes do at open. The nesting bound is part of that check, so a stored payload nested past it is refused at load too, where it used to load and refuse on read. - A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). - `#[cultcache(index)]` means unique, as in C# `RefuseIndexSharing`: within one type and one alias no two records hold one non-blank value. A put, batch, push or load that would make two records hold a value is refused, naming both keys and the alias, and a store that already holds a duplicate does not open. A grouping index is no longer expressible; a consumer that relied on shared values must drop the `index` and look up by another way. diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 59ffa1af..522e226e 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -600,14 +600,22 @@ impl std::error::Error for PayloadTooDeep {} /// Walks the MessagePack headers of one payload without recursion, counting every array and map and /// refusing the first container past [`MAX_PAYLOAD_DEPTH`]. Scalars are skipped by length and nothing -/// is allocated per element. A payload that ends early is the decoder's to refuse. +/// is allocated per element. Only the first value is walked; a payload that ends early is the +/// decoder's to refuse. A depth-only probe: [`decode_payload`] also refuses bytes after the value. pub fn check_payload_depth(bytes: &[u8]) -> std::result::Result<(), PayloadTooDeep> { + first_value_end(bytes).map(|_| ()) +} + +/// Where the first MessagePack value of `bytes` ends, walking only that value (its tail is never +/// read). `None` when the payload ends inside a header; a final scalar cut short ends past the +/// payload's length, which is not bytes after the value either. Both are the decoder's to refuse. +fn first_value_end(bytes: &[u8]) -> std::result::Result, PayloadTooDeep> { let mut remaining = [0usize; MAX_PAYLOAD_DEPTH]; let mut depth = 0usize; let mut position = 0usize; while position < bytes.len() { let Some((advance, children)) = element_header(bytes, position) else { - return Ok(()); + return Ok(None); }; position = position.saturating_add(advance); if let Some(children) = children { @@ -629,7 +637,18 @@ pub fn check_payload_depth(bytes: &[u8]) -> std::result::Result<(), PayloadTooDe depth -= 1; } if depth == 0 { - return Ok(()); + return Ok(Some(position)); + } + } + Ok(None) +} + +/// The one rule for "is this payload one value": the nesting bound, then no bytes after the first +/// value. The count is all it reports; the tail is never walked or quoted. +fn check_record_payload(bytes: &[u8]) -> Result<()> { + if let Some(end) = first_value_end(bytes)? { + if end < bytes.len() { + bail!("payload has {} bytes after its first MessagePack value", bytes.len() - end); } } Ok(()) @@ -784,7 +803,7 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re grown(&mut reader, |grown| T::deserialize(grown)) } -/// The one decode of a document payload: the nesting bound first, then [`decode_grown`], whose +/// The one decode of a document payload: the payload check first (nesting bound, one value), then [`decode_grown`], whose /// rmp-serde depth counter (1024) is only a backstop. A failure keeps its kind and never quotes the /// bytes. Readers of stored envelope payloads outside this crate decode through it too. /// @@ -796,7 +815,7 @@ fn decode_grown(bytes: &[u8]) -> std::result::Re /// verified on x86_64-pc-windows-msvc by hand (`decode_stack`, revision 10a39dac, 13 tests, debug and release); a /// `stacker` bump or a change to the stack-size tests repeats it. pub fn decode_payload(payload: &[u8]) -> Result { - check_payload_depth(payload)?; + check_record_payload(payload)?; decode_grown(payload).map_err(opaque_decode_error) } @@ -918,10 +937,11 @@ impl Drop for UnreadScope { } } -/// [`decode_payload`] with the mark: the same depth check and grown decode, inside a scope the derives' slot -/// visitors note into. A payload past the nesting limit is refused before the decode, never marked. +/// [`decode_payload`] with the mark: the same payload check and grown decode, inside a scope the derives' slot +/// visitors note into. A payload past the nesting limit or with bytes after its value is refused before the +/// decode, never marked. pub fn decode_payload_marked(payload: &[u8]) -> Result<(T, Option)> { - check_payload_depth(payload)?; + check_record_payload(payload)?; let scope = UnreadScope::open(); let value = decode_grown(payload).map_err(opaque_decode_error)?; Ok((value, scope.mark())) @@ -948,10 +968,10 @@ struct Definition { unread_slots: fn(&[u8]) -> Result>, } -/// The nesting bound for an envelope admitted without a typed decode, from the same walk that -/// [`decode_payload`] runs first. Names the entry's type and key, never the payload. -fn require_payload_depth(entry: &CultCacheEnvelope) -> Result<()> { - check_payload_depth(&entry.payload).with_context(|| { +/// The payload check for an envelope admitted without a typed decode, the one [`decode_payload`] +/// runs first. Names the entry's type and key, never the payload. +fn require_record_payload(entry: &CultCacheEnvelope) -> Result<()> { + check_record_payload(&entry.payload).with_context(|| { format!( "refused CultCache envelope {:?} at key {:?}", entry.r#type, entry.key @@ -2917,8 +2937,30 @@ pub struct CultCache { /// Records a store holds that no registered type claims: (store index, the envelope as the store read it). Derived from the last /// pull and replaced with it; written nowhere else. It decides only refusals at its keys. foreign: Vec<(usize, CultCacheEnvelope)>, + /// Why the last pull of the backing stores was refused; `None` once a pull succeeds. While set, every write and removal is refused. + load_refused: Option, } +/// A write or removal refused because the cache's last load of its backing stores was refused. The cache holds no +/// image of those stores, so any write would persist an image that lacks the records it never read. Repair the store +/// named by `reason` (which names a type, a key and a clause, never a payload) and load again. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CultCacheLoadRefused { + pub reason: String, +} + +impl std::fmt::Display for CultCacheLoadRefused { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "CultCache refuses writes and removals: its load of the backing stores failed ({}); repair the store and load again", + self.reason + ) + } +} + +impl std::error::Error for CultCacheLoadRefused {} + /// A record a store holds that no registered type claims: carried, never served. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ForeignRecord { @@ -2980,6 +3022,7 @@ impl CultCache { projectors: BTreeMap::new(), stores: Vec::new(), foreign: Vec::new(), + load_refused: None, } } @@ -3116,12 +3159,39 @@ impl CultCache { /// Reads every store into a staged image and replaces the cache's view /// only when all of them loaded, so a refused load admits nothing. pub fn pull_all_backing_stores(&mut self) -> Result<()> { + match self.stage_pull() { + Ok((loaded, foreign, holders)) => { + self.entries = loaded; + self.foreign = foreign; + self.index_holders = holders; + self.load_refused = None; + Ok(()) + } + Err(error) => { + self.load_refused = Some(format!("{error:#}")); + Err(error) + } + } + } + + /// The one gate of every write and removal: a cache whose load was refused holds no image of its stores. + fn require_loaded(&self) -> Result<()> { + match &self.load_refused { + Some(reason) => Err(CultCacheLoadRefused { reason: reason.clone() }.into()), + None => Ok(()), + } + } + + /// The image every store holds, judged whole, installed nowhere. + #[allow(clippy::type_complexity)] + fn stage_pull(&self) -> Result<(BTreeMap<(String, String), CultCacheEnvelope>, Vec<(usize, CultCacheEnvelope)>, IndexHolders)> { let mut loaded = BTreeMap::new(); let mut foreign = Vec::new(); let known_types: BTreeSet = self.definitions.keys().cloned().collect(); let schema_name_definitions = self.schema_name_definitions.clone(); for (index, registration) in self.stores.iter().enumerate() { for mut entry in registration.store.pull_all()? { + require_record_payload(&entry)?; let Some(canonical_type) = resolve_registered_type( &known_types, &schema_name_definitions, @@ -3154,10 +3224,7 @@ impl CultCache { } self.refuse_second_globals(&loaded)?; let holders = self.judge_image(&loaded)?; - self.entries = loaded; - self.foreign = foreign; - self.index_holders = holders; - Ok(()) + Ok((loaded, foreign, holders)) } /// The records the last pull found in a store that no registered type claims, sorted by key (UTF-16 order). The cache does not @@ -3383,6 +3450,7 @@ impl CultCache { } pub fn put(&mut self, key: impl Into, value: &T) -> Result { + self.require_loaded()?; let (entry, parsed) = self.prepare_entry(key, value)?; self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; @@ -3460,6 +3528,7 @@ impl CultCache { } pub fn put_prepared_batch(&mut self, staged: Vec) -> Result<()> { + self.require_loaded()?; if staged.is_empty() { return Err(anyhow!("CultCache batch must contain at least one entry")); } @@ -3484,7 +3553,7 @@ impl CultCache { entry.r#type )); } - require_payload_depth(entry)?; + require_record_payload(entry)?; self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_claimed_id(entry)?; let Some(home) = self.home_index(&entry.r#type)? else { @@ -3527,6 +3596,7 @@ impl CultCache { } pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { + self.require_loaded()?; let parsed = self.validate_envelope::(&entry)?; self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_claimed_id(&entry)?; @@ -3547,6 +3617,7 @@ impl CultCache { /// admission; CultCache still enforces registered type, identity, timestamp, /// the payload nesting bound ([`MAX_PAYLOAD_DEPTH`]), routing, and persistence. pub fn put_raw_envelope(&mut self, entry: CultCacheEnvelope) -> Result<()> { + self.require_loaded()?; if !self.definitions.contains_key(&entry.r#type) { return Err(anyhow!( "No entry type registered for CultCache envelope type {:?}", @@ -3565,7 +3636,7 @@ impl CultCache { entry.r#type )); } - require_payload_depth(&entry)?; + require_record_payload(&entry)?; self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_claimed_id(&entry)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; @@ -3621,6 +3692,7 @@ impl CultCache { } pub fn delete(&mut self, key: &str) -> Result { + self.require_loaded()?; self.require_entry_type::()?; self.refuse_unread_slots(T::TYPE, key)?; let id = entry_id_parts(T::TYPE, key); @@ -3662,6 +3734,7 @@ impl CultCache { entry.r#type )); } + require_record_payload(&entry)?; self.home_index(&entry.r#type)?; self.refuse_claimed_id(&entry)?; self.refuse_retype(&loaded, &self.foreign, &entry)?; @@ -6296,6 +6369,34 @@ mod tests { Ok(()) } + #[test] + fn put_raw_envelope_refuses_bytes_after_the_value() -> Result<()> { + let temp = tempfile::tempdir()?; + let store_path = temp.path().join("tail.cc"); + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + let tailed = CultCacheEnvelope { + key: "app".to_string(), + r#type: Settings::TYPE.to_string(), + payload: vec![0x91, 0x01, 0xc0], + stored_at: now_utc_second(), + schema_id: None, + }; + let error = format!("{:#}", cache.put_raw_envelope(tailed.clone()).unwrap_err()); + assert!(error.contains("payload has 1 bytes after its first MessagePack value"), "{error}"); + assert!(error.contains("app") && error.contains(Settings::TYPE), "{error}"); + assert!(cache.snapshot().is_empty()); + assert!(!store_path.exists() || SingleFileMessagePackBackingStore::new(&store_path).pull_all()?.is_empty()); + let error = format!("{:#}", cache.put_prepared_batch(vec![tailed.clone()]).unwrap_err()); + assert!(error.contains("payload has 1 bytes after its first MessagePack value"), "{error}"); + assert!(cache.snapshot().is_empty()); + // The same bytes without the tail are admitted: the refusal is the count, not the shape. + cache.put_raw_envelope(CultCacheEnvelope { payload: vec![0x91, 0x01], ..tailed })?; + assert_eq!(cache.snapshot().len(), 1); + Ok(()) + } + #[test] fn soa_snapshot_round_trips_registered_entries() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/packages/cultcache-rs/tests/slot_readonly.rs b/packages/cultcache-rs/tests/slot_readonly.rs index 5bffcd9a..0031e372 100644 --- a/packages/cultcache-rs/tests/slot_readonly.rs +++ b/packages/cultcache-rs/tests/slot_readonly.rs @@ -481,9 +481,11 @@ fn load_soa_refuses_a_same_length_replacement_of_a_marked_record() { let (mut cache, _dir, _) = held_marked_and_clean(); let all = cache.snapshot_envelopes(); let marked = all.iter().find(|held| held.key == "marked").unwrap(); - // The clean record's bytes padded to the marked record's length: same length, different bytes. - let mut same_length = vec![0xc0; marked.payload.len()]; - same_length[..2].copy_from_slice(&[0x91, 0xa1]); + // One value (a bin16) of the marked record's length: same length, different bytes. A value followed by padding + // would be refused at admission as bytes after the value, before the read-only check is reached. + let length = marked.payload.len(); + let mut same_length = vec![0u8; length]; + same_length[..3].copy_from_slice(&[0xc5, ((length - 3) >> 8) as u8, (length - 3) as u8]); assert_eq!(same_length.len(), marked.payload.len()); let mut replaced = all.clone(); replaced.iter_mut().find(|held| held.key == "marked").unwrap().payload = same_length; @@ -491,3 +493,47 @@ fn load_soa_refuses_a_same_length_replacement_of_a_marked_record() { assert_refused_with(&error, "marked", &mark("Composition", 3, 2, 1)); assert_eq!(cache.snapshot_envelopes(), all); } + +// The payload check runs before the mark scope opens: a tail on a slot-bearing payload is refused by count, never marked. +#[test] +fn a_tail_on_a_slot_payload_is_refused_before_any_mark() { + let mut tailed = vector("extra-slot-record.msgpack"); + tailed.push(0xc0); + let error = decode_payload_marked::(&tailed).unwrap_err(); + assert!(format!("{error:#}").contains("payload has 1 bytes after its first MessagePack value"), "{error:#}"); + // Nothing leaked: the next marked decode finds exactly its own mark. + let found = decode_payload_marked::(&vector("extra-slot-record.msgpack")).unwrap().1.unwrap(); + assert_eq!(found.occurrences, 1); +} + +// A store holding marked records and one with a tail is refused at load; the cache then refuses every write with the load +// refusal (not a per-key mark conflict), and the store file is untouched. +#[test] +fn a_store_with_marked_records_and_a_tail_is_refused_and_its_cache_writes_nothing() { + use cultcache_rs::CacheBackingStore; + use cultcache_rs::CultCacheLoadRefused; + let (mut held, _dir, path) = held_marked_and_clean(); + let mut tailed = vector("union-surface.msgpack"); + tailed.push(0xc0); + SingleFileMessagePackBackingStore::new(&path).push(&envelope("tailed", tailed)).unwrap(); + let before = std::fs::read(&path).unwrap(); + // The cache that held the store re-reads it and is refused. + assert!(format!("{:#}", held.pull_all_backing_stores().unwrap_err()).contains("bytes after")); + let fresh_value = constructed(); + let mut fresh = registered(); + fresh.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + assert!(fresh.pull_all_backing_stores().is_err()); + for cache in [&mut held, &mut fresh] { + let staged = cache.prepare_entry("new", &fresh_value).unwrap().0; + let errors = [ + cache.put("new", &fresh_value).unwrap_err(), + cache.put_prepared_batch(vec![staged.clone()]).unwrap_err(), + cache.put_raw_envelope(staged).unwrap_err(), + cache.delete::("new").unwrap_err(), + ]; + for error in errors { + assert!(error.downcast_ref::().is_some(), "{error:#}"); + } + } + assert_eq!(std::fs::read(&path).unwrap(), before); +} diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index c51bd3d8..67f2b389 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -4,9 +4,13 @@ use cultcache_rs::CultCache; use cultcache_rs::CacheBackingStore; use cultcache_rs::CultCacheEnvelope; +use cultcache_rs::CultCacheLoadRefused; +use cultcache_rs::CultCacheSoaTable; use cultcache_rs::MAX_PAYLOAD_DEPTH; use cultcache_rs::PayloadTooDeep; use cultcache_rs::check_payload_depth; +use cultcache_rs::decode_payload; +use cultcache_rs::RedbMessagePackBackingStore; use cultcache_rs::DatabaseEntry; use cultcache_rs::SingleFileMessagePackBackingStore; @@ -464,7 +468,7 @@ fn envelope(key: &str, payload: &[u8]) -> CultCacheEnvelope { } // A payload past the bound cannot be admitted through the cache, so a store file written behind its -// back stands in for one a peer or an older runtime left. +// back stands in for one a peer or an older runtime left; load or read refuses it. fn through_cache(payload: &[u8]) -> Result { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("foreign.cc"); @@ -472,7 +476,10 @@ fn through_cache(payload: &[u8]) -> Result { let mut cache = CultCache::new(); cache.register_entry_type::().unwrap(); cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); - cache.pull_all_backing_stores().unwrap(); + // The payload check runs at load, so a payload past the bound is refused there, before any read. + if let Err(error) = cache.pull_all_backing_stores() { + return Err(format!("{error:#}")); + } match cache.get::("k") { Ok(value) => Ok(value.expect("the envelope was stored")), Err(error) => Err(format!("{error:#}")), @@ -529,6 +536,157 @@ fn bounds_vectors() { assert_eq!((accepted, refused, read_only), (4, 7, 1)); } +// A payload is one MessagePack value (bytes-after-outcomes.msgpack): every door refuses a tail by +// its count, names the clause, never the depth limit (the tail is not walked) and never a value. +#[test] +fn bytes_after_vectors() { + let rows: Vec = rmp_serde::from_slice(&vector("bytes-after-outcomes.msgpack")).unwrap(); + assert_eq!(rows.len(), 4); + let surface = vector("union-surface.msgpack"); + for row in &rows { + let fields = row.as_array().unwrap(); + let (file, outcome, union, detail) = (fields[0].as_str().unwrap(), fields[1].as_str().unwrap(), fields[2].as_str().unwrap(), fields[4].as_str().unwrap()); + assert_eq!((outcome, union, detail), ("refused", "UnionSurface", "bytes-after"), "{file}"); + let bytes = vector(file); + assert_eq!(&bytes[..surface.len()], &surface[..], "{file}"); + let clause = format!("payload has {} bytes after its first MessagePack value", bytes.len() - surface.len()); + let check = |door: &str, message: &str| { + assert!(message.contains(&clause), "{file} {door}: {message}"); + assert!(!message.contains("the limit is"), "{file} {door} walked the tail: {message}"); + assert!(!message.contains("CANARY-bytes-after-7d1e"), "{file} {door} echoed the tail: {message}"); + assert!(!message.contains(CANARY), "{file} {door} echoed a value: {message}"); + }; + let error = decode_payload::(&bytes).unwrap_err(); + assert!(error.downcast_ref::().is_none(), "{file}"); + check("decode_payload", &format!("{error:#}")); + // A store written behind the cache's back is refused at load, naming the type, the key and the count. + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("foreign.cc"); + SingleFileMessagePackBackingStore::new(&path).push(&envelope("k", &bytes)).unwrap(); + let mut single = CultCache::new(); + single.register_entry_type::().unwrap(); + single.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + let message = format!("{:#}", single.pull_all_backing_stores().unwrap_err()); + check("single-file load", &message); + assert!(message.contains("\"k\"") && message.contains(UnionSurface::TYPE), "{file}: {message}"); + let redb_path = temp.path().join("foreign.redb"); + RedbMessagePackBackingStore::new(&redb_path).unwrap().push(&envelope("k", &bytes)).unwrap(); + let mut redb = CultCache::new(); + redb.register_entry_type::().unwrap(); + redb.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb_path).unwrap()).unwrap(); + let message = format!("{:#}", redb.pull_all_backing_stores().unwrap_err()); + check("redb load", &message); + assert!(message.contains("\"k\"") && message.contains(UnionSurface::TYPE), "{file}: {message}"); + let mut soa = CultCache::new(); + soa.register_entry_type::().unwrap(); + let message = format!("{:#}", soa.load_soa(CultCacheSoaTable::from_envelopes(vec![envelope("k", &bytes)])).unwrap_err()); + check("load_soa", &message); + assert!(message.contains("\"k\"") && message.contains(UnionSurface::TYPE), "{file}: {message}"); + assert!(soa.snapshot().is_empty(), "{file}"); + // Raw admission refuses and stages nothing. + let mut raw = CultCache::new(); + raw.register_entry_type::().unwrap(); + let message = format!("{:#}", raw.put_raw_envelope(envelope("k", &bytes)).unwrap_err()); + check("put_raw_envelope", &message); + assert!(raw.snapshot().is_empty(), "{file}"); + } +} + +// Depth is found inside the first value, so a tail never hides it: a payload nested past the bound +// with a tail after it is the depth refusal, not bytes after. +#[test] +fn depth_beats_a_tail() { + let mut deep = vec![0x91u8; MAX_PAYLOAD_DEPTH + 1]; + deep.push(0xc0); + deep.push(0xc0); + let error = decode_payload::(&deep).unwrap_err(); + assert_eq!(error.downcast_ref::(), Some(&TOO_DEEP), "{error:#}"); +} + +// Depth is refused where the payload is loaded: a store holding one nested past the bound is not read. +#[test] +fn depth_is_refused_at_load() { + let mut deep = vec![0x91u8; MAX_PAYLOAD_DEPTH + 1]; + deep.push(0xc0); + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("deep.cc"); + SingleFileMessagePackBackingStore::new(&path).push(&envelope("k", &deep)).unwrap(); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + let error = cache.pull_all_backing_stores().unwrap_err(); + assert_eq!(error.downcast_ref::(), Some(&TOO_DEEP), "{error:#}"); + assert!(cache.snapshot().is_empty()); +} + +// A cache whose load was refused refuses every write and removal and leaves the store as it was: a +// later write would otherwise persist the empty image over the records it never read. +#[test] +fn a_refused_load_refuses_every_write() { + let tail = { + let mut bytes = vector("union-surface.msgpack"); + bytes.push(0xc0); + bytes + }; + let good = envelope("good", &vector("union-surface.msgpack")); + let bad = envelope("bad", &tail); + for redb in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join(if redb { "seed.redb" } else { "seed.cc" }); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + if redb { + let mut store = RedbMessagePackBackingStore::new(&path).unwrap(); + store.push(&good).unwrap(); + store.push(&bad).unwrap(); + drop(store); + cache.add_generic_backing_store(RedbMessagePackBackingStore::new(&path).unwrap()).unwrap(); + } else { + let mut store = SingleFileMessagePackBackingStore::new(&path); + store.push(&good).unwrap(); + store.push(&bad).unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + } + let load = cache.pull_all_backing_stores().unwrap_err(); + assert!(format!("{load:#}").contains("bytes after"), "{load:#}"); + let before = std::fs::read(&path).unwrap(); + let value = constructed(); + let staged = cache.prepare_entry("new", &value).unwrap().0; + let results: Vec<(&str, anyhow::Error)> = vec![ + ("put", cache.put("new", &value).unwrap_err()), + ("put_prepared_batch", cache.put_prepared_batch(vec![staged.clone()]).unwrap_err()), + ("put_envelope", cache.put_envelope::(staged.clone()).unwrap_err()), + ("put_raw_envelope", cache.put_raw_envelope(staged.clone()).unwrap_err()), + ("update", cache.update::("good", |current| current.unwrap_or_else(constructed)).unwrap_err()), + ("delete", cache.delete::("good").unwrap_err()), + ("delete a key the image lacks", cache.delete::("absent").unwrap_err()), + ]; + for (door, error) in results { + let refusal = error.downcast_ref::().unwrap_or_else(|| panic!("{door} redb={redb}: {error:#}")); + assert!(refusal.to_string().contains("bytes after"), "{door}: {refusal}"); + } + assert_eq!(std::fs::read(&path).unwrap(), before, "redb={redb}: the store changed"); + assert!(cache.snapshot().is_empty()); + // Repairing the store and loading again lifts it. + if redb { + RedbMessagePackBackingStore::new(&path).unwrap().delete(&bad).unwrap(); + } else { + SingleFileMessagePackBackingStore::new(&path).delete(&bad).unwrap(); + } + cache.pull_all_backing_stores().unwrap(); + cache.put("new", &value).unwrap(); + } +} + +// A payload that ends early is not bytes after the value: the decoder refuses it, not the check. +#[test] +fn a_truncated_payload_is_the_decoders_not_bytes_after() { + let mut truncated = vec![0x91, 0xd9, 0x40]; + truncated.extend_from_slice(b"short"); + let message = format!("{:#}", decode_payload::(&truncated).unwrap_err()); + assert!(!message.contains("bytes after"), "{message}"); +} + fn wrap_chain(wraps: usize, leaf: Node) -> UnionSurface { let mut node = leaf; for _ in 0..wraps { diff --git a/packages/cultnet-rs/CHANGELOG.md b/packages/cultnet-rs/CHANGELOG.md index 384395ec..1a01dbe1 100644 --- a/packages/cultnet-rs/CHANGELOG.md +++ b/packages/cultnet-rs/CHANGELOG.md @@ -15,5 +15,6 @@ only in git history. ### Breaking +- A CultNet frame with bytes after its message is refused, by count (through `cultcache_rs::decode_payload`); the decoder used to read the message and drop the tail. No producer emits one. Behaviour change: the next release is a minor bump (0.x breaking). - `CultNetDocumentRegistry::register` returns `Result<&mut Self>` where it returned `&mut Self`. A binding that claims a document type or a schema id another binding already holds is refused with `cultcache_rs::SchemaConflictError` (naming the new binding's schema id and both document types) and nothing is bound; binding a type to the schema id it already holds still replaces that binding. Callers chain `register(...)?`. - `CultMeshCdnChunkRef` gains a public `id` field, the decimal text of `offset` (the reference's element id, slot 4). A manifest written before ids reads `""`; encoding writes the slot. diff --git a/packages/cultnet-rs/tests/cultnet.rs b/packages/cultnet-rs/tests/cultnet.rs index 64c57aaf..7bc84957 100644 --- a/packages/cultnet-rs/tests/cultnet.rs +++ b/packages/cultnet-rs/tests/cultnet.rs @@ -1810,12 +1810,10 @@ fn a_replicated_payload_nested_past_the_bound_is_refused_typed_on_apply() -> Res }; assert_eq!(documents.len(), 1); - backing.push(&envelope("past", limit + 1))?; - let mut cache = CultCache::new(); - cache.register_entry_type::()?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store))?; - cache.pull_all_backing_stores()?; - let past = registry.create_raw_snapshot_response(&cache, "past", None, Some(&["past".to_string()]))?; + // The sender cannot load a payload past the bound, so the frame is the at-limit one with its bytes swapped. + let mut past = at_limit.clone(); + let CultNetMessage::SnapshotResponseRaw { documents, .. } = &mut past else { unreachable!() }; + documents[0].payload = envelope("past", limit + 1).payload; let mut target = CultCache::new(); target.register_entry_type::()?; let error = registry diff --git a/packages/cultnet-rs/tests/decode_door.rs b/packages/cultnet-rs/tests/decode_door.rs index fd5bab6a..c55a6404 100644 --- a/packages/cultnet-rs/tests/decode_door.rs +++ b/packages/cultnet-rs/tests/decode_door.rs @@ -190,6 +190,16 @@ fn message_past_the_bound_is_refused_typed() { }); } +// A frame is one message: cultcache's payload rule refuses bytes after it, by count. +#[test] +fn frame_with_bytes_after_is_refused() { + let mut bytes = hello_with(&nested(0)); + assert!(decode_message(bytes.clone(), BIG).is_ok()); + bytes.push(0xc0); + let error = format!("{:#}", decode_message(bytes, BIG).unwrap_err()); + assert!(error.contains("1 bytes after its first MessagePack value"), "{error}"); +} + fn raw_v1_bytes(header_unknown: Option<&[u8]>) -> Vec { let mut header: Vec<(rmpv::Value, rmpv::Value)> = vec![ ("schemaId".into(), "s".into()), From 66bea107d5ce61e2871b3ca3dedd4d9c5cc6ec51 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 06:02:51 +0000 Subject: [PATCH 08/15] TypeScript: a payload is one MessagePack value (bytes-after-ts) (#130) --- packages/cultcache-ts/CHANGELOG.md | 2 + packages/cultcache-ts/README.md | 6 +- .../cultcache-ts/src/cult-cache-inspector.ts | 18 ++- packages/cultcache-ts/src/document.ts | 8 +- packages/cultcache-ts/src/payload-depth.ts | 39 ++++-- packages/cultcache-ts/src/wire.ts | 5 +- .../cultcache-ts/test/payload-depth.test.ts | 20 ++++ .../cultcache-ts/test/union-parity.test.ts | 113 +++++++++++++++++- .../test/provider-session-wire.test.ts | 13 ++ packages/cultnet-ts/test/content.test.ts | 7 ++ packages/cultnet-ts/test/cultnet.test.ts | 23 ++++ 11 files changed, 229 insertions(+), 25 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index b91a0535..2da49fe9 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -24,10 +24,12 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Added +- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. The bytes-after refusal is internal (`checkPayloadIsOneValue`, not exported); no new public name. - `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal. ### Changed +- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication, the provider command), and the inspector refuse bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse. The refused set is unchanged; only the text changed (it was `@msgpack`'s `RangeError` "Extra N of M byte(s)" for a plain document and the codec reader's "bytes after its value" for a codec document, and the inspector's preview no longer shows a tail), so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Stated limit: a definition with its own `formatter` is not walked, so whether its payload is one value is the formatter's. Implied bump: 0.16.0 (pre-1.0 minor, with the Breaking entries above). - Opening a legacy (pre-v1) store is a pure read: payloads that were not bytes are normalized in memory and the file is left as it was, so a codec that refuses the content cannot lose the old bytes. The next push, delete or flush rewrites the file as v1, merging what the open read. diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 6fea306a..a081cd92 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -49,7 +49,9 @@ through `decodeDocumentPayload`, and when you need the bytes themselves (hashing write) take them with `documentPayloadBytes`, the sanctioned escape. - `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter, - codec or plain map and then its schema. The definition is required. + codec or plain map and then its schema. The definition is required. A payload is exactly one + MessagePack value: bytes after the first are refused by count, naming the type, and never walked or + echoed (a payload that ends early is the decoder's to refuse). - `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter, codec or plain map. - `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a @@ -327,7 +329,7 @@ If you do not supply one, `CultCacheTS` uses MessagePack plus Zod validation by The 256-container nesting bound (`MAX_PAYLOAD_DEPTH`, refused with `CultCachePayloadDepthError`) covers the MessagePack the library itself writes: the default format and a codec. A payload your formatter writes is opaque bytes in any format, so the -library does not walk it as MessagePack; its format and its bounds are your formatter's. +library does not walk it as MessagePack; its format, its bounds and whether it is one value are your formatter's. Zod is no longer mandatory. Any schema object with a `parse(...)` function will do, which means generated Ajv-backed JSON-schema contracts can feed the cache diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index 1c359157..71643913 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -1,6 +1,6 @@ import { decode } from "@msgpack/msgpack"; -import { checkPayloadDepth } from "./payload-depth"; +import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth"; import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format"; export interface CultCacheInspection { @@ -235,12 +235,22 @@ function decodeLegacyEnvelope(value: unknown): InspectedRecord { }; } -// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it. +// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it, +// and bytes after the first value are refused with the shared clause (the tail is never decoded or shown). function decodeBoundedPayload(payload: Uint8Array): unknown { - checkPayloadDepth(payload); + checkPayloadIsOneValue(payload, "document payload"); return decode(payload); } +// Where the first value ends (the whole payload when it ends early or cannot be walked): a preview never shows a tail. +function valueEnd(payload: Uint8Array): number { + try { + return checkPayloadDepth(payload) ?? payload.length; + } catch { + return payload.length; + } +} + function previewPayload(payload: Uint8Array): Pick { try { return { payloadPreview: toJsonSafe(decodeBoundedPayload(payload)) }; @@ -248,7 +258,7 @@ function previewPayload(payload: Uint8Array): Pick value.toString(16).padStart(2, "0")) .join(" "), }, diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 8d295bd4..cb1ef589 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -1,6 +1,6 @@ import type { NotedSlots } from "./codec"; import { decodeMarked, resolveDocumentFormatter } from "./document-formatter"; -import { checkPayloadDepth } from "./payload-depth"; +import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth"; import type { AnyCultCacheDocumentDefinition, CultCacheSchemaCatalogMember, @@ -129,7 +129,11 @@ export function decodeDocumentPayloadMarked; unreadSlots?: NotedSlots } { const bytes = documentPayloadBytes(payload); - if (writesLibraryMessagePack(definition)) checkPayloadDepth(bytes, `document type "${definition.type}" payload`); + if (writesLibraryMessagePack(definition)) { + // A payload is exactly one MessagePack value: bytes after the first are refused by count before any formatter reads, + // so a tail is refused before any slot is marked. + checkPayloadIsOneValue(bytes, `document type "${definition.type}" payload`); + } const { value: decoded, unreadSlots } = decodeMarked(definition, bytes); const value = definition.schema.parse(decoded) as CultCacheDocumentValue; return unreadSlots ? { value, unreadSlots } : { value }; diff --git a/packages/cultcache-ts/src/payload-depth.ts b/packages/cultcache-ts/src/payload-depth.ts index 40ca7afe..45b8d5ca 100644 --- a/packages/cultcache-ts/src/payload-depth.ts +++ b/packages/cultcache-ts/src/payload-depth.ts @@ -12,8 +12,9 @@ export class CultCachePayloadDepthError extends Error { // Walks the headers of one MessagePack value iteratively (a count stack bounded by the limit, scalars skipped by length, // nothing read) and throws at the first container past the limit. The message names the subject, the depth and the limit, -// never payload bytes. A payload that ends early is the reader's to refuse. -export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): void { +// never payload bytes. Returns the offset where the first value ends, or undefined when the payload ends early (the +// reader's to refuse); what follows that offset is never walked, and whether it may exist is the caller's rule. +export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): number | undefined { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const remaining = new Float64Array(MAX_PAYLOAD_DEPTH); let depth = 0; @@ -33,12 +34,12 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code <= 0x8f) { container = true; header = 1; children = 2 * (code & 0x0f); } else if (code <= 0x9f) { container = true; header = 1; children = code & 0x0f; } else if (code <= 0xbf) skip = 1 + (code & 0x1f); - else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return; skip = 2 + n; } - else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return; skip = 5 + n; } - else if (code === 0xc7) { const n = width(1); if (n === undefined) return; skip = 3 + n; } - else if (code === 0xc8) { const n = width(2); if (n === undefined) return; skip = 4 + n; } - else if (code === 0xc9) { const n = width(4); if (n === undefined) return; skip = 6 + n; } + else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return undefined; skip = 2 + n; } + else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return undefined; skip = 5 + n; } + else if (code === 0xc7) { const n = width(1); if (n === undefined) return undefined; skip = 3 + n; } + else if (code === 0xc8) { const n = width(2); if (n === undefined) return undefined; skip = 4 + n; } + else if (code === 0xc9) { const n = width(4); if (n === undefined) return undefined; skip = 6 + n; } else if (code === 0xca || code === 0xce || code === 0xd2) skip = 5; else if (code === 0xcb || code === 0xcf || code === 0xd3) skip = 9; else if (code === 0xcc || code === 0xd0) skip = 2; @@ -47,10 +48,10 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload else if (code === 0xd6) skip = 6; else if (code === 0xd7) skip = 10; else if (code === 0xd8) skip = 18; - else if (code === 0xdc) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = n; } - else if (code === 0xdd) { const n = width(4); if (n === undefined) return; container = true; header = 5; children = n; } - else if (code === 0xde) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = 2 * n; } - else { const n = width(4); if (n === undefined) return; container = true; header = 5; children = 2 * n; } + else if (code === 0xdc) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = n; } + else if (code === 0xdd) { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = n; } + else if (code === 0xde) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = 2 * n; } + else { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = 2 * n; } if (container) { if (depth === MAX_PAYLOAD_DEPTH) throw new CultCachePayloadDepthError(MAX_PAYLOAD_DEPTH + 1, MAX_PAYLOAD_DEPTH, subject); @@ -63,8 +64,20 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload position += skip; } + // A scalar whose declared length runs past the payload is an early end, not a value. + if (position > bytes.length) return undefined; // One element is complete; close every container it was the last element of. while (depth > 0 && --remaining[depth - 1] === 0) depth--; - if (depth === 0) return; + if (depth === 0) return position; } + return undefined; +} + +// The one rule that a payload is exactly one MessagePack value: the depth walk, then a refusal of bytes after the first +// value, by count, naming the subject, from the walk's end. The tail is never walked or echoed; a payload that ends early +// stays the decoder's to refuse. decodeDocumentPayload and the inspector speak this clause and no other. +export function checkPayloadIsOneValue(bytes: Uint8Array, subject: string): number | undefined { + const end = checkPayloadDepth(bytes, subject); + if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`); + return end; } diff --git a/packages/cultcache-ts/src/wire.ts b/packages/cultcache-ts/src/wire.ts index 12bf572a..d351c446 100644 --- a/packages/cultcache-ts/src/wire.ts +++ b/packages/cultcache-ts/src/wire.ts @@ -128,7 +128,6 @@ export function readWire(payload: Uint8Array): unknown { } } - const tree = next(); - if (offset !== payload.length) throw new Error("the MessagePack payload has bytes after its value."); - return tree; + // One value is read; the caller owns what follows it (decodeDocumentPayload refuses a payload with a tail). + return next(); } diff --git a/packages/cultcache-ts/test/payload-depth.test.ts b/packages/cultcache-ts/test/payload-depth.test.ts index 382f0672..6fb81265 100644 --- a/packages/cultcache-ts/test/payload-depth.test.ts +++ b/packages/cultcache-ts/test/payload-depth.test.ts @@ -119,6 +119,19 @@ test("a payload that ends early, is empty or carries trailing bytes is the reade assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0))); }); +test("the walk returns where the first value ends, and undefined when the payload ends early", () => { + assert.equal(checkPayloadDepth(bytes(0xc0)), 1); + assert.equal(checkPayloadDepth(bytes(0xc0, 0xc1, 0xc1)), 1, "a top-level scalar ends at its own length; the tail is not judged"); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01, 0xa1, 0x61, 0xc0)), 5 - 1, "a container ends when its last child completes"); + assert.equal(checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), 1, "the tail is never walked"); + assert.equal(checkPayloadDepth(bytes()), undefined); + assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined); + assert.equal(checkPayloadDepth(bytes(0xd9)), undefined); + for (const cut of [[0xcc], [0xcb, 0x3f, 0xf8], [0xa5, 0x61, 0x62, 0x63], [0xc4, 0x02, 0x01], [0xd7, 0x01]]) { + assert.equal(checkPayloadDepth(bytes(...cut)), undefined, `a scalar cut short: ${cut.map((value) => value.toString(16)).join(" ")}`); + } +}); + test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => { refused(arrays(100_000), "100000 nested arrays"); refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps"); @@ -198,6 +211,13 @@ test("the inspector previews a payload at the limit and reports one past it, or } }); +test("the inspector refuses bytes after a payload's first value with the shared clause and previews no tail", async () => { + const file = await storeWith(bytes(0xc0, 0xc1, 0xc1)); + const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!; + assert.equal(record.payloadDecodeError, "document payload has 2 bytes after its first MessagePack value."); + assert.deepEqual(record.payloadPreview, { bytes: 3, hexPrefix: "c0" }); +}); + // A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not. const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index d688cb74..d917ea7a 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -338,7 +338,7 @@ test("the wire reader keeps 64-bit integers whole, tells a float from an integer assert.throws(() => readWire(fromHex("dd")), /truncated/u); assert.throws(() => readWire(fromHex("ddffffffff")), /truncated/u, "a huge array header reserves nothing"); assert.throws(() => readWire(fromHex("a5616263")), /truncated/u); - assert.throws(() => readWire(fromHex("0102")), /bytes after its value/u); + assert.equal(readWire(fromHex("0102")), 1, "the reader reads one value; the document owner refuses what follows"); assert.throws(() => readWire(fromHex("81a161a162")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("c1")), /map, extension or reserved/u); assert.throws(() => readWire(fromHex("d4010a")), /map, extension or reserved/u); @@ -725,3 +725,114 @@ test("a cache load refuses a store whose record is past the limit, naming the ty const refusing = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(await read("depth-past-limit.msgpack"))])).build(); await assert.rejects(() => refusing.pullAllBackingStores(), (error: Error) => error instanceof CultCachePayloadDepthError && !error.message.includes(CANARY)); }); + +// ---- bytes-after vectors: a payload is exactly one MessagePack value (tests/vectors/union-parity/README.md) ---- + +// The C# reference writes the vectors: union-surface.msgpack followed by a tail the first value does not own. The one +// owner refuses by count, naming the type, from the walk's end; it never walks or echoes the tail. +async function bytesAfterEntries() { + const raw = decode(await read("bytes-after-outcomes.msgpack")) as unknown[][]; + return raw.map(([file, outcome, unionName, key, detail]) => ({ file: file as string, outcome: outcome as string, unionName: unionName as string, key: key as number | null, detail: detail as string })); +} + +// The C# vector writer's canary (BytesAfterVectorTests): it sits in the deep tail, so an echo of the tail would carry it. +const TAIL_CANARY = "CANARY-bytes-after-7d1e"; +const valueLength = async () => (await read("union-surface.msgpack")).length; +const plainDocument = () => defineDocumentType({ type: "tests.plain_surface", schema: z.unknown() }); +const codecDocument = () => defineDocumentType({ type: "tests.codec_surface", schema: surfaceSchema, codec: surfaceCodec }); + +async function assertBytesAfterRefused(file: string, tail: number, run: () => unknown | Promise, type: string) { + const clause = `${type}`; + await assert.rejects( + async () => run(), + (error: Error) => { + assert.ok(!(error instanceof CultCachePayloadDepthError), `${file}: the tail was walked`); + assert.ok(!error.message.includes(CANARY) && !error.message.includes(TAIL_CANARY), `${file}: the refusal echoed the tail`); + assert.ok(!error.message.includes("the limit is"), `${file}: the depth refusal`); + assert.ok(error.message.includes(clause), `${file}: names the type: ${error.message}`); + assert.ok(error.message.includes(`${tail} bytes after its first MessagePack value`), `${file}: names the count: ${error.message}`); + return true; + }, + file, + ); +} + +test("every bytes-after vector is refused by the one owner, by count, on the codec, plain and store paths", async () => { + const entries = await bytesAfterEntries(); + assert.equal(entries.length, 4); + const length = await valueLength(); + const surface = surfaceDocument(); + const plain = plainDocument(); + const codecOnly = codecDocument(); + const dir = await mkdtemp(join(tmpdir(), "bytes-after-")); + for (const { file, outcome, unionName, detail } of entries) { + assert.equal(outcome, "refused"); + assert.equal(unionName, "UnionSurface"); + assert.equal(detail, "bytes-after"); + const bytes = await read(file); + const tail = bytes.length - length; + assert.ok(tail > 0, file); + if (file === "bytes-after-deep-tail.msgpack") assert.ok(Buffer.from(bytes).includes(TAIL_CANARY), "the deep tail carries the canary the assertions look for"); + const payload = documentPayloadFromBytes(bytes); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(surface, payload), surface.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(plain, payload), plain.type); + await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(codecOnly, payload), codecOnly.type); + + // The store path: a single-file store holding the vector as a record payload refuses on load. + const path = join(dir, `${file}.cc`); + await new SingleFileMessagePackBackingStore(path).push(envelopeOf(bytes)); + const cache = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type); + + // The put path: an envelope arriving with the tail is refused and held nowhere. + const written: CultCacheEnvelope[] = []; + const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build(); + await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type); + assert.equal(written.length, 0, `${file}: nothing reached the store`); + assert.equal(held.get(surface, "surface"), undefined, `${file}: nothing is held in memory`); + } +}); + +test("a payload with an unread slot and a tail is refused by count before any slot is marked or the record held", async () => { + const withSlot = await read("extra-slot-record.msgpack"); + const surface = surfaceDocument(); + const tailed = Uint8Array.from([...withSlot, 0xc0]); + const whole = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(withSlot)])).build(); + await whole.pullAllBackingStores(); + assert.ok(whole.unreadSlots(surface, "surface"), "without a tail the record is held with its unread slots"); + + const loading = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(tailed)])).build(); + await assertBytesAfterRefused("extra-slot-record+tail", 1, () => loading.pullAllBackingStores(), surface.type); + assert.equal(loading.get(surface, "surface"), undefined); + assert.equal(loading.unreadSlots(surface, "surface"), undefined); + + const written: CultCacheEnvelope[] = []; + const putting = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build(); + await assertBytesAfterRefused("extra-slot-record+tail", 1, () => putting.putEnvelope(surface, envelopeOf(tailed)), surface.type); + assert.equal(written.length, 0); + assert.equal(putting.get(surface, "surface"), undefined); + assert.equal(putting.unreadSlots(surface, "surface"), undefined); +}); + +test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => { + const whole = await read("union-surface.msgpack"); + const surface = surfaceDocument(); + for (const cut of [0, 1, 10, whole.length - 1]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(whole.slice(0, cut))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `truncated at ${cut}`, + ); + } + // A header cut short of its length is an early end too, whatever width it declares. + for (const early of ["dc00", "dd000000", "de00", "c4", "d9", "a5616263", "9291"]) { + assert.throws( + () => decodeDocumentPayload(surface, documentPayloadFromBytes(fromHex(early))), + (error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError), + `early end ${early}`, + ); + } + const opaque = defineDocumentType({ type: "tests.opaque", schema: z.unknown(), formatter: { encode: (value: Uint8Array) => value, decode: (bytes: Uint8Array) => bytes } }); + const tailed = await read("bytes-after-nil.msgpack"); + assert.equal((decodeDocumentPayload(opaque, documentPayloadFromBytes(tailed)) as Uint8Array).length, tailed.length); +}); diff --git a/packages/cultmesh-ts/test/provider-session-wire.test.ts b/packages/cultmesh-ts/test/provider-session-wire.test.ts index 4905f161..69314fbe 100644 --- a/packages/cultmesh-ts/test/provider-session-wire.test.ts +++ b/packages/cultmesh-ts/test/provider-session-wire.test.ts @@ -83,3 +83,16 @@ test("a provider command that is not a map, or lacks a required field, is refuse (error: Error) => error.message === "CultMesh provider command must be a MessagePack map.", ); }); + +test("a provider command payload with bytes after its first value is refused by count, without echoing the tail", () => { + const command = { commandId: "c1", commandKind: "k", providerId: "p", serviceInstanceId: "s", payload: { n: 1 } }; + const whole = documentPayloadBytes(encodeDocumentPayload(providerCommandDocument, command)); + const tailed = documentPayloadFromBytes(Uint8Array.from([...whole, ...Buffer.from("canary-tail-8c1f")])); + assert.throws( + () => decodeProviderCommandDocument({ + schemaId: cultMeshProviderSessionSchemas.command, recordKey: "p/s/c1", storedAt: "2026-10-11T00:00:00Z", + payloadEncoding: "messagepack" as const, payload: tailed, + }), + (error: Error) => error.message.includes("16 bytes after its first MessagePack value") && !error.message.includes("canary-tail-8c1f"), + ); +}); diff --git a/packages/cultnet-ts/test/content.test.ts b/packages/cultnet-ts/test/content.test.ts index ec23b64e..e2ae5ed9 100644 --- a/packages/cultnet-ts/test/content.test.ts +++ b/packages/cultnet-ts/test/content.test.ts @@ -392,6 +392,13 @@ test("decodeManifest accepts and refuses exactly what the reference does, and re } }); +test("a manifest with bytes after its first value is refused by count, whatever the tail is", () => { + const whole = encodeManifest(manifestOf()); + for (const tail of [[0xc0], [0xc1, 0xc1], [0x90]]) { + assert.throws(() => decodeManifest(Uint8Array.from([...whole, ...tail])), new RegExp(`${tail.length} bytes after its first MessagePack value`)); + } +}); + test("a manifest written as a map names the manifest, not a field inside it", () => { assert.throws(() => decodeManifest(encode({ 0: "a", 9: { k: "v" } })), /field 'manifest' has the wrong type/); }); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index ac4f539d..581fd66a 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -14,6 +14,8 @@ import { SchemaConflictError, SingleFileMessagePackBackingStore, defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, schemaIdentityOf, } from "@gamecult/cultcache-ts"; @@ -3140,3 +3142,24 @@ test("CultNet registry names the canonical schema id of a compatible alias, and assert.equal(registry.canonicalSchemaId("p3.nobody"), undefined); assert.equal(registry.canonicalSchemaId("p3.shared"), undefined, "an id several bindings claim resolves to none"); }); + +test("a replicated record with bytes after its first MessagePack value is refused by count and held nowhere", async () => { + const definition = defineDocumentType({ type: "tails.note", schemaId: "tails.note.v1", schema: z.object({ body: z.string() }) }); + const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition })]); + const origin = CultCache.builder().withDocumentType(definition).build(); + await origin.put(definition, "k", { body: "x" }); + const document = registry.createRawSnapshotResponse(origin, "tails").documents[0]!; + const whole = documentPayloadBytes(document.payload); + const target = CultCache.builder().withDocumentType(definition).build(); + await assert.rejects( + registry.applyRawDocumentPutMessage(target, { + schemaVersion: "cultnet.document_put_raw.v0", + messageId: "tails", + document: { ...document, payload: documentPayloadFromBytes(Uint8Array.from([...whole, 0xc0, 0xc1])) }, + }), + (error: Error) => error.message.includes('document type "tails.note" payload has 2 bytes after its first MessagePack value'), + ); + assert.deepEqual(target.snapshot(), []); + await registry.applyRawDocumentPutMessage(target, { schemaVersion: "cultnet.document_put_raw.v0", messageId: "whole", document }); + assert.deepEqual(target.getRequired(definition, "k"), { body: "x" }); +}); From 12f3801893956272a26d61413ea98301251e7171 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 08:01:32 +0000 Subject: [PATCH 09/15] Rust: a cache must pull its stores before it writes (cache-must-pull-before-write) (#133) --- packages/cultcache-rs/CHANGELOG.md | 1 + packages/cultcache-rs/src/lib.rs | 170 +++++++++++++++++-- packages/cultcache-rs/tests/slot_readonly.rs | 2 + packages/cultcache-rs/tests/union_parity.rs | 2 + 4 files changed, 160 insertions(+), 15 deletions(-) diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index 4e97588c..c6d72492 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -16,6 +16,7 @@ Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0). - A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it. - The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check. - A cache whose last `pull_all_backing_stores` was refused (a tail, a payload past the nesting bound, a duplicate unique-index value, a retype conflict, a record from the wrong store) refuses every `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch`, `update` and `delete` with `CultCacheLoadRefused` until a pull succeeds. Before, it stayed writable over an empty image, and the next batch write replaced the store with only the new records. +- A cache with a backing store also refuses every write and removal, with `CultCacheLoadRefused`, until it has pulled: a cache that never pulled, or that gained a store since its last pull, holds no image of its stores, and a batch from it replaced a populated store with only the new records. `is_writable` answers false meanwhile. A cache with no store needs no pull. Callers that attach a store and write (cultmesh nodes created with `pull_on_start: false`) must call `pull_all_backing_stores` first. - `decode_payload`, `put_raw_envelope` and prepared batches refuse a payload with bytes after its first MessagePack value (`payload has {n} bytes after its first MessagePack value`), by count; the tail is never walked or quoted. They used to read the first value and keep or drop the tail, so a record with a stray tail was served and rewritten without it. A store holding such a record (single-file, redb, `load_soa`) is refused at load, naming the type, the key and the count, and stays unreadable until the record is repaired; the same check now runs on every record at load, as the C# and Python runtimes do at open. The nesting bound is part of that check, so a stored payload nested past it is refused at load too, where it used to load and refuse on read. - A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 522e226e..47a3dd18 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -2937,13 +2937,24 @@ pub struct CultCache { /// Records a store holds that no registered type claims: (store index, the envelope as the store read it). Derived from the last /// pull and replaced with it; written nowhere else. It decides only refusals at its keys. foreign: Vec<(usize, CultCacheEnvelope)>, - /// Why the last pull of the backing stores was refused; `None` once a pull succeeds. While set, every write and removal is refused. - load_refused: Option, + /// Whether the cache holds an image of its stores. While it is not `Pulled`, every write and removal is refused. + load: LoadState, } -/// A write or removal refused because the cache's last load of its backing stores was refused. The cache holds no -/// image of those stores, so any write would persist an image that lacks the records it never read. Repair the store -/// named by `reason` (which names a type, a key and a clause, never a payload) and load again. +/// What the cache knows of its stores. `Unpulled` is the state of a cache that has never pulled, or that gained a store +/// since: its image lacks whatever the stores hold, and a batch would replace them with it. +#[derive(Clone, Debug, PartialEq, Eq)] +enum LoadState { + Unpulled, + Pulled, + /// Why the last pull was refused. + Refused(String), +} + +/// A write or removal refused because the cache holds no image of its backing stores: its last load was refused, or it has +/// not pulled since its stores were attached. Any write would persist an image that lacks the records it never read. +/// Pull; for a refused load, first repair the store named by `reason` (which names a type, a key and a clause, never a +/// payload). #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultCacheLoadRefused { pub reason: String, @@ -2953,7 +2964,7 @@ impl std::fmt::Display for CultCacheLoadRefused { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( formatter, - "CultCache refuses writes and removals: its load of the backing stores failed ({}); repair the store and load again", + "CultCache refuses writes and removals: it holds no image of its backing stores ({}); repair the store if it was refused and pull again", self.reason ) } @@ -3022,7 +3033,7 @@ impl CultCache { projectors: BTreeMap::new(), stores: Vec::new(), foreign: Vec::new(), - load_refused: None, + load: LoadState::Unpulled, } } @@ -3128,6 +3139,10 @@ impl CultCache { store: Box::new(store), types, }); + let loaded_before = self.load.clone(); + if self.load == LoadState::Pulled { + self.load = LoadState::Unpulled; + } // A record's home cannot change after it is admitted. let candidate = self.stores.len() - 1; let held: BTreeSet = self.entries.keys().map(|(type_id, _)| type_id.clone()).collect(); @@ -3146,6 +3161,7 @@ impl CultCache { self.describe_store(after) ); self.stores.pop(); + self.load = loaded_before; return Err(error); } } @@ -3164,21 +3180,27 @@ impl CultCache { self.entries = loaded; self.foreign = foreign; self.index_holders = holders; - self.load_refused = None; + self.load = LoadState::Pulled; Ok(()) } Err(error) => { - self.load_refused = Some(format!("{error:#}")); + self.load = LoadState::Refused(format!("{error:#}")); Err(error) } } } /// The one gate of every write and removal: a cache whose load was refused holds no image of its stores. + /// A cache without stores has nothing to lose and needs no pull. fn require_loaded(&self) -> Result<()> { - match &self.load_refused { - Some(reason) => Err(CultCacheLoadRefused { reason: reason.clone() }.into()), - None => Ok(()), + match &self.load { + _ if self.stores.is_empty() => Ok(()), + LoadState::Pulled => Ok(()), + LoadState::Refused(reason) => Err(CultCacheLoadRefused { reason: reason.clone() }.into()), + LoadState::Unpulled => Err(CultCacheLoadRefused { + reason: "pull_all_backing_stores has not run since the stores were attached".to_string(), + } + .into()), } } @@ -3269,9 +3291,12 @@ impl CultCache { /// Whether a write or removal of `(type_id, key)` would be admitted: false when the held payload carries slots past /// its registered type's declared slots, at any depth, or cannot be read by this build at all (the record is - /// read-only), when a store carries a foreign record at the key, or when another registered type holds it. True for a + /// read-only), when the cache holds no image of its stores (see [`CultCacheLoadRefused`]), when a store carries a foreign record at the key, or when another registered type holds it. True for a /// key nothing holds. An unregistered type is an error. Advisory: the write entries ask the same checks themselves. pub fn is_writable(&self, type_id: &str, key: &str) -> Result { + if self.require_loaded().is_err() { + return Ok(false); + } if !matches!(self.held_finding(type_id, key)?, HeldFinding::Free) { return Ok(false); } @@ -4724,6 +4749,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?; + cache.pull_all_backing_stores()?; records(&mut cache) } @@ -4965,6 +4991,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_backing_store(SingleFileMessagePackBackingStore::new(&routed), [Settings::TYPE])?; + cache.pull_all_backing_stores()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic))?; cache.pull_all_backing_stores()?; let (staged, _) = cache.prepare_entry("other", &settings_one())?; @@ -5066,6 +5093,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put("shared", &settings_one())?; let before = std::fs::read(&store_path)?; @@ -5098,6 +5126,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put("kept", &settings_one())?; let before = std::fs::read(&store_path)?; @@ -5206,6 +5235,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; let (settings, _) = cache.prepare_entry( "app", @@ -5912,6 +5942,7 @@ mod tests { inner: SingleFileMessagePackBackingStore::new(&store_path), refuse_batch: refusal.clone(), })?; + cache.pull_all_backing_stores()?; cache.put( "app", &Settings { @@ -5952,6 +5983,7 @@ mod tests { ["settings"], )?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.pull_all_backing_stores()?; cache.put( "app", @@ -5980,6 +6012,7 @@ mod tests { let temp = tempfile::tempdir()?; let mut cache = CultCache::new(); cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("a.cc")))?; + cache.pull_all_backing_stores()?; let error = cache .add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("b.cc"))) .unwrap_err(); @@ -5993,6 +6026,7 @@ mod tests { let temp = tempfile::tempdir()?; let mut cache = CultCache::new(); cache.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("a.cc")), ["settings"])?; + cache.pull_all_backing_stores()?; let error = cache .add_backing_store( SingleFileMessagePackBackingStore::new(temp.path().join("b.cc")), @@ -6002,6 +6036,7 @@ mod tests { assert!(error.to_string().contains("already routed"), "{error}"); // A refused registration claims nothing, so "note" is still free. cache.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("c.cc")), ["note"])?; + cache.pull_all_backing_stores()?; assert_eq!(cache.stores.len(), 2); Ok(()) } @@ -6014,6 +6049,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.pull_all_backing_stores()?; cache.put("app", &Settings { theme: "v1-generic".to_string(), retries: 1 })?; let error = cache @@ -6043,15 +6079,17 @@ mod tests { let mut writer = CultCache::new(); writer.register_entry_type::()?; writer.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + writer.pull_all_backing_stores()?; writer.put("app", &Settings { theme: "stray".to_string(), retries: 0 })?; let generic_bytes = fs::read(&generic_path)?; let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"])?; - cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.pull_all_backing_stores()?; // The cache already holds a record, in its home store, before the refused load. cache.put("held", &Settings { theme: "home".to_string(), retries: 0 })?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; let before = cache.snapshot(); let settings_bytes = fs::read(&settings_path)?; let error = cache.pull_all_backing_stores().unwrap_err(); @@ -6075,7 +6113,9 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), [Settings::TYPE])?; + cache.pull_all_backing_stores()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.pull_all_backing_stores()?; cache.put("app", &Settings { theme: "before".to_string(), retries: 1 })?; let before = cache.snapshot(); @@ -6117,6 +6157,7 @@ mod tests { let mut homeless = CultCache::new(); homeless.register_registry(TestEntries)?; homeless.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("settings.cc")), ["settings"])?; + homeless.pull_all_backing_stores()?; homeless.put("held", &Settings { theme: "held".to_string(), retries: 0 })?; let before = homeless.snapshot(); let error = homeless.load_soa(table).unwrap_err(); @@ -6133,7 +6174,9 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"])?; + cache.pull_all_backing_stores()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?; + cache.pull_all_backing_stores()?; cache.put("memo", &Note { title: "generic".to_string(), body: "home".to_string() })?; cache.put("old", &Settings { theme: "kept".to_string(), retries: 0 })?; @@ -6164,6 +6207,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put("app", &Settings { theme: "before".to_string(), retries: 1 })?; let before = cache.snapshot(); let store_bytes = fs::read(&store_path)?; @@ -6203,6 +6247,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; let updated = cache.update::("app", |current| { let mut current = current.unwrap_or(Settings { @@ -6225,6 +6270,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put( "app", &Settings { @@ -6288,6 +6334,7 @@ mod tests { let mut origin = CultCache::new(); origin.register_entry_type::()?; origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?; + origin.pull_all_backing_stores()?; origin.put( "app", &Settings { @@ -6301,6 +6348,7 @@ mod tests { let mut target = CultCache::new(); target.register_entry_type::()?; target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?; + target.pull_all_backing_stores()?; let applied = target.put_envelope::(envelope.clone())?; assert_eq!( @@ -6334,6 +6382,7 @@ mod tests { cache.register_entry_type::()?; cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put_raw_envelope(envelope.clone())?; assert_eq!(cache.snapshot(), vec![envelope.clone()]); @@ -6376,6 +6425,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; let tailed = CultCacheEnvelope { key: "app".to_string(), r#type: Settings::TYPE.to_string(), @@ -6404,6 +6454,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put( "app", &Settings { @@ -6441,6 +6492,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_registry(TestEntries)?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put( "settings:a", &Settings { @@ -6523,6 +6575,7 @@ mod tests { let mut cache = CultCache::new(); cache.register_entry_type::()?; cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?; + cache.pull_all_backing_stores()?; cache.put( "primary", &Settings { @@ -8051,4 +8104,91 @@ mod tests { assert_eq!(fs::read(&path)?, before); Ok(()) } -} \ No newline at end of file + + /// A store already holding `Settings:"held"`, then a fresh cache attached to it that has not pulled. + fn unpulled_cache_over_populated_stores() -> Result<(tempfile::TempDir, Vec, Vec)> { + let temp = tempfile::tempdir()?; + let single = temp.path().join("single.cc"); + let redb = temp.path().join("redb.db"); + let held = Settings { theme: "held".to_string(), retries: 1 }; + let mut seeder = CultCache::new(); + seeder.register_registry(TestEntries)?; + seeder.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&single))?; + seeder.pull_all_backing_stores()?; + seeder.put("held", &held)?; + let mut seeder = CultCache::new(); + seeder.register_registry(TestEntries)?; + seeder.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb)?)?; + seeder.pull_all_backing_stores()?; + seeder.put("held", &held)?; + + let mut over_single = CultCache::new(); + over_single.register_registry(TestEntries)?; + over_single.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&single))?; + let mut over_redb = CultCache::new(); + over_redb.register_registry(TestEntries)?; + over_redb.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb)?)?; + Ok((temp, vec![over_single, over_redb], vec![single, redb])) + } + + #[test] + fn a_cache_that_never_pulled_refuses_every_write_and_removal() -> Result<()> { + let (_temp, caches, _paths) = unpulled_cache_over_populated_stores()?; + for mut cache in caches { + let fresh = Note { title: "t".to_string(), body: "b".to_string() }; + let (staged, _) = cache.prepare_entry("fresh", &fresh)?; + for error in [ + cache.put_prepared_batch(vec![staged.clone()]).unwrap_err(), + cache.put("fresh", &fresh).unwrap_err(), + cache.put_raw_envelope(staged.clone()).unwrap_err(), + cache.delete::("held").unwrap_err(), + ] { + assert!(error.downcast_ref::().is_some(), "{error:#}"); + } + assert!(cache.snapshot().is_empty()); + } + Ok(()) + } + + #[test] + fn a_batch_from_a_cache_that_never_pulled_cannot_wipe_the_store() -> Result<()> { + let (_temp, caches, paths) = unpulled_cache_over_populated_stores()?; + for (mut cache, path) in caches.into_iter().zip(paths) { + let (staged, _) = cache.prepare_entry("fresh", &Note { title: "t".to_string(), body: "b".to_string() })?; + let _ = cache.put_prepared_batch(vec![staged]); + let mut reader = CultCache::new(); + reader.register_registry(TestEntries)?; + if path.extension().is_some_and(|extension| extension == "db") { + reader.add_generic_backing_store(RedbMessagePackBackingStore::new(&path)?)?; + reader.pull_all_backing_stores()?; + } else { + reader.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?; + reader.pull_all_backing_stores()?; + } + reader.pull_all_backing_stores()?; + assert_eq!(reader.get_required::("held")?.theme, "held", "{path:?}"); + } + Ok(()) + } + + #[test] + fn pulling_lifts_the_gate_and_a_store_attached_later_closes_it_again() -> Result<()> { + let (temp, mut caches, _paths) = unpulled_cache_over_populated_stores()?; + let mut cache = caches.remove(0); + cache.pull_all_backing_stores()?; + cache.put("fresh", &Note { title: "t".to_string(), body: "b".to_string() })?; + assert_eq!(cache.get_required::("held")?.theme, "held"); + + let mut routed = CultCache::new(); + routed.register_registry(TestEntries)?; + let settings = Settings { theme: "t".to_string(), retries: 0 }; + routed.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("routed.cc")), [Settings::TYPE])?; + routed.pull_all_backing_stores()?; + routed.put("fresh", &settings)?; + routed.add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("other.cc")))?; + assert!(routed.put("again", &settings).unwrap_err().downcast_ref::().is_some()); + routed.pull_all_backing_stores()?; + routed.put("again", &settings)?; + Ok(()) + } +} diff --git a/packages/cultcache-rs/tests/slot_readonly.rs b/packages/cultcache-rs/tests/slot_readonly.rs index 0031e372..cea0853a 100644 --- a/packages/cultcache-rs/tests/slot_readonly.rs +++ b/packages/cultcache-rs/tests/slot_readonly.rs @@ -149,6 +149,7 @@ fn every_read_only_vector_decodes_marked_and_is_not_writable() { let mut cache = registered(); let dir = tempfile::tempdir().unwrap(); cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc"))).unwrap(); + cache.pull_all_backing_stores().unwrap(); cache.put_raw_envelope(envelope("k", payload.clone())).unwrap(); assert_eq!(cache.get::("k").unwrap(), Some(constructed()), "{file}"); assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap(), "{file}"); @@ -470,6 +471,7 @@ fn is_writable_is_false_for_a_type_with_no_home_store() { let mut cache = registered(); // One store, routed to SlotUnit only: UnionSurface has no home. cache.add_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc")), [SlotUnit::TYPE]).unwrap(); + cache.pull_all_backing_stores().unwrap(); assert!(cache.put("k", &constructed()).is_err()); assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap()); assert!(cache.is_writable(SlotUnit::TYPE, "k").unwrap()); diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index 67f2b389..bc3aba6d 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -718,6 +718,7 @@ fn write_refused_past_limit() { let mut cache = CultCache::new(); cache.register_entry_type::().unwrap(); cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); let at_limit = wrap_chain(126, boxed("0000000000f0", vec![])); assert_eq!(cache.put("at", &at_limit).unwrap(), at_limit); @@ -801,6 +802,7 @@ fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() { cache.register_entry_type::().unwrap(); cache.register_entry_type::().unwrap(); cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); cache.put_raw_envelope(raw(type_name, "at-raw", at.clone())).unwrap(); cache.put_prepared_batch(vec![raw(type_name, "at-batch", at.clone())]).unwrap(); let stored = cache.snapshot_envelopes(); From fbeb773d35412fb02ed29ca2fdb260a9c0c52686 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 08:45:52 +0000 Subject: [PATCH 10/15] Rust: batch writes merge onto the store (rs-batch-merges-onto-store) (#135) --- packages/cultcache-rs/CHANGELOG.md | 1 + packages/cultcache-rs/README.md | 7 +- packages/cultcache-rs/src/lib.rs | 167 ++++++++++++++++++++--------- 3 files changed, 121 insertions(+), 54 deletions(-) diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index c6d72492..b43c13c6 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -16,6 +16,7 @@ Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0). - A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it. - The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check. - A cache whose last `pull_all_backing_stores` was refused (a tail, a payload past the nesting bound, a duplicate unique-index value, a retype conflict, a record from the wrong store) refuses every `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch`, `update` and `delete` with `CultCacheLoadRefused` until a pull succeeds. Before, it stayed writable over an empty image, and the next batch write replaced the store with only the new records. +- `CacheBackingStore::push_all` writes the given entries onto the store as it is when the write lands, under the store's lock or transaction: a record under the same identity is replaced and no other record is removed. It used to replace everything the store held, so a cache whose image was stale (another writer had added a record since its pull) wiped that record with `put_prepared_batch`. `put_prepared_batch` now hands the store its staged records alone, as C# `CommitBatch` and Python `push_all` do; the lay-back of foreign records is gone because they are never rewritten. A custom `CacheBackingStore` must give `push_all` merge semantics. `push_all` reads the store, so through a dangling link it is refused like `push` and no longer replaces the link; a soft flush is unchanged. - A cache with a backing store also refuses every write and removal, with `CultCacheLoadRefused`, until it has pulled: a cache that never pulled, or that gained a store since its last pull, holds no image of its stores, and a batch from it replaced a populated store with only the new records. `is_writable` answers false meanwhile. A cache with no store needs no pull. Callers that attach a store and write (cultmesh nodes created with `pull_on_start: false`) must call `pull_all_backing_stores` first. - `decode_payload`, `put_raw_envelope` and prepared batches refuse a payload with bytes after its first MessagePack value (`payload has {n} bytes after its first MessagePack value`), by count; the tail is never walked or quoted. They used to read the first value and keep or drop the tail, so a record with a stray tail was served and rewritten without it. A store holding such a record (single-file, redb, `load_soa`) is refused at load, naming the type, the key and the count, and stays unreadable until the record is repaired; the same check now runs on every record at load, as the C# and Python runtimes do at open. The nesting bound is part of that check, so a stored payload nested past it is refused at load too, where it used to load and refuse on read. - A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index ed9701d5..da4185e2 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -384,10 +384,9 @@ an empty store. Any other failure to reach it, such as a symlink loop, a file where a parent directory should be, or a missing permission, is an I/O error. A store file that exists and holds zero bytes is `Undecodable`: no CultCache writer leaves one. A symbolic link whose target is gone is not a missing store: -reading it is an I/O error, so a read-then-write (`push`, `delete`, the exchanges) -fails before writing. Writes do not yet follow links: a write through a live link, -and a `push_all` through a dangling one, replace the link with a regular file and -leave its target alone. A refusal names no +reading it is an I/O error, so a read-then-write (`push`, `push_all`, `delete`, the exchanges) +fails before writing. Writes do not yet follow links: a write through a live link +replaces the link with a regular file and leaves its target alone. A refusal names no value read from the file; it names a record only by its key and schema id, and a header only when it has the shape `cultcache.store.v`. diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 47a3dd18..678ff4d6 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -1253,12 +1253,14 @@ pub trait CacheBackingStore: Send { fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()>; fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()>; - /// Replaces everything the store holds with `entries` in one atomic step: - /// on `Err` the store still holds exactly what it held before. A cache - /// batch (`put_prepared_batch`) is all-or-nothing only through this. There - /// is no default, because a replace assembled from `delete` and `push` - /// calls leaves the store emptied or partial when one of them fails; a - /// store without an atomic replace does not compile: + /// Writes `entries` onto the store as it is when the write lands, in one + /// atomic step: a record under the same identity is replaced and no other + /// record is removed or changed, so a writer whose image is stale cannot + /// drop a record it never saw. On `Err` the store still holds exactly what + /// it held before. A cache batch (`put_prepared_batch`) is all-or-nothing + /// only through this. There is no default, because a write assembled from + /// `push` calls leaves the store partial when one of them fails; a store + /// without an atomic write does not compile: /// /// ```compile_fail,E0046 /// use cultcache_rs::{CacheBackingStore, CultCacheEnvelope}; @@ -1675,13 +1677,7 @@ impl SingleFileMessagePackBackingStore { // payloads this store passes through may hold ids), one that is gone or legacy is written v1. The file is read by // the same reader as an open, so a store this runtime cannot read completely, a variant included, is refused // and never overwritten. - let disk = match self.read_store_unlocked() { - // A dangling link holds no store, so a whole-file replace has no header to keep and replaces the link (R3 pins - // this). The only NotFound the reader returns is that one: a merging writer has already refused it when it - // read the current entries. - Err(error) if io_not_found(&error) => DiskStore::absent(), - other => other?, - }; + let disk = self.read_store_unlocked()?; let entries = settle_one_record_per_key(entries, &disk)?; let bytes = encode_store_snapshot(&entries, &disk) .and_then(|snapshot| rmp_serde::to_vec(&snapshot).context("failed to encode MessagePack")) @@ -1962,14 +1958,18 @@ impl CacheBackingStore for SingleFileMessagePackBackingStore { } // Of entries sharing a key and a schema id the last one given is the record, whatever their type names (a type - // renamed under a stable id); a second schema id under a key reaches the commit guard. + // renamed under a stable id); it replaces the held record under that key and schema id. A second schema id + // under a key reaches the commit guard, which judges the merged store. let mut by_id = BTreeMap::<(String, String), &CultCacheEnvelope>::new(); for entry in entries { by_id.insert((schema_identity(entry).to_string(), entry.key.clone()), entry); } - let mut entries: Vec = by_id.into_values().cloned().collect(); - entries.sort_by_key(entry_id); - self.write_all_unlocked(&entries) + let mut merged = self.read_all_unlocked()?; + let replaced: BTreeSet<_> = by_id.values().map(|entry| entry_id(entry)).collect(); + merged.retain(|held| !replaced.contains(&entry_id(held))); + merged.extend(by_id.into_values().cloned()); + merged.sort_by_key(entry_id); + self.write_all_unlocked(&merged) }) } } @@ -2757,13 +2757,6 @@ impl CacheBackingStore for OwnedRedbMessagePackBackingStore { let write = self.inner.database.begin_write()?; { let mut table = write.open_table(REDB_ENVELOPES)?; - let keys = table - .iter()? - .map(|row| Ok(row?.0.value().to_vec())) - .collect::>>()?; - for key in keys { - table.remove(key.as_slice())?; - } for entry in entries { insert_redb_entry(&mut table, entry)?; } @@ -2899,13 +2892,6 @@ impl CacheBackingStore for RedbMessagePackBackingStore { let write = database.begin_write()?; { let mut table = write.open_table(REDB_ENVELOPES)?; - let keys = table - .iter()? - .map(|row| Ok(row?.0.value().to_vec())) - .collect::>>()?; - for key in keys { - table.remove(key.as_slice())?; - } for entry in entries { insert_redb_entry(&mut table, entry)?; } @@ -3602,18 +3588,12 @@ impl CultCache { } let change = self.judge_unique(&staged.iter().collect::>())?; if let Some(store_index) = store_index { - // The home store receives its own records only: those it already holds in this cache plus the staged ones, and - // the records it carries that no registered type claims: push_all replaces the store's records, so a foreign - // record is laid back as the store read it. - let snapshot: Vec<_> = next - .values() - .filter(|entry| self.resolve_route_index(&entry.r#type) == Some(store_index)) - .chain(self.foreign.iter().filter(|(home, _)| *home == store_index).map(|(_, entry)| entry)) - .cloned() - .collect(); + // The home store is handed the staged records alone and applies them onto what it holds now. + // A key staged twice lands as its last record, whatever the store. + let landing: BTreeMap<_, _> = staged.iter().map(|entry| (entry_id(entry), entry.clone())).collect(); self.stores[store_index] .store - .push_all(&snapshot, PushAllOptions::default())?; + .push_all(&landing.into_values().collect::>(), PushAllOptions::default())?; } self.entries = next; change.apply(&mut self.index_holders); @@ -4324,12 +4304,6 @@ fn release_lock(lock: File, result: Result) -> Result { result } -fn io_not_found(error: &anyhow::Error) -> bool { - error - .chain() - .any(|cause| cause.downcast_ref::().is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound)) -} - /// True only when nothing is at `path`; a symbolic link, dangling or not, is something. /// A path that cannot be inspected (a file where a directory should be, no permission) /// is an error, never absent. @@ -5400,6 +5374,12 @@ mod tests { Ok(()) } + fn io_not_found(error: &anyhow::Error) -> bool { + error + .chain() + .any(|cause| cause.downcast_ref::().is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound)) + } + fn test_envelope(r#type: &str, key: &str, payload: &[u8]) -> CultCacheEnvelope { CultCacheEnvelope { key: key.to_string(), @@ -7980,15 +7960,17 @@ mod tests { #[cfg(unix)] #[test] - fn r3_decides_resolve_or_refuse_a_push_all_through_a_dangling_link_replaces_the_link() -> Result<()> { + fn a_push_all_through_a_dangling_link_is_refused_like_push_and_writes_nothing() -> Result<()> { let temp = tempfile::tempdir()?; let volume = temp.path().join("volume"); fs::create_dir(&volume)?; let link = temp.path().join("store.cc"); std::os::unix::fs::symlink(volume.join("store.cc"), &link)?; - SingleFileMessagePackBackingStore::new(&link).push_all(&[snapshot_envelope("a", b"1")], PushAllOptions::default())?; - assert!(fs::symlink_metadata(&link)?.file_type().is_file(), "the link was replaced by a regular file"); + let mut store = SingleFileMessagePackBackingStore::new(&link); + assert!(store.push_all(&[snapshot_envelope("a", b"1")], PushAllOptions::default()).is_err()); + assert!(store.push(&snapshot_envelope("a", b"1")).is_err()); + assert!(fs::symlink_metadata(&link)?.file_type().is_symlink(), "the link is left as it was"); assert_eq!(fs::read_dir(&volume)?.count(), 0, "nothing reached the link's volume"); Ok(()) } @@ -8191,4 +8173,89 @@ mod tests { routed.put("again", &settings)?; Ok(()) } + + #[test] + fn a_stale_cache_batch_keeps_the_records_another_writer_added() -> Result<()> { + let temp = tempfile::tempdir()?; + let single = temp.path().join("single.cc"); + let redb = temp.path().join("redb.db"); + let open = |path: &Path| -> Result { + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + if path.extension().is_some_and(|extension| extension == "db") { + cache.add_generic_backing_store(RedbMessagePackBackingStore::new(path)?)?; + } else { + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?; + } + cache.pull_all_backing_stores()?; + Ok(cache) + }; + for path in [&single, &redb] { + let mut seed = open(path)?; + seed.put("held", &Settings { theme: "held".to_string(), retries: 1 })?; + let mut stale = open(path)?; + let mut other = open(path)?; + other.put("held", &Settings { theme: "other-newer".to_string(), retries: 3 })?; + other.put("added", &Settings { theme: "other".to_string(), retries: 2 })?; + let (staged, _) = stale.prepare_entry("mine", &Note { title: "t".to_string(), body: "b".to_string() })?; + stale.put_prepared_batch(vec![staged])?; + let reader = open(path)?; + assert_eq!(reader.get_required::("added")?.theme, "other", "{path:?}"); + assert_eq!(reader.get_required::("held")?.theme, "other-newer", "a record another writer updated is not reverted"); + assert_eq!(reader.get_required::("mine")?.title, "t", "{path:?}"); + } + Ok(()) + } + + #[test] + fn every_store_merges_push_all_and_lands_the_last_of_a_repeated_key() -> Result<()> { + let temp = tempfile::tempdir()?; + let stores: Vec> = vec![ + Box::new(SingleFileMessagePackBackingStore::new(temp.path().join("single.cc"))), + Box::new(RedbMessagePackBackingStore::new(temp.path().join("redb.db"))?), + Box::new(OwnedRedbMessagePackBackingStore::new(temp.path().join("owned.db"))?), + ]; + for mut store in stores { + store.push(&test_envelope("pushall.merge", "kept", b"kept"))?; + store.push_all(&[test_envelope("pushall.merge", "new", b"new")], PushAllOptions::default())?; + let keys: Vec<_> = store.pull_all()?.into_iter().map(|entry| entry.key).collect(); + assert_eq!(keys, ["kept", "new"]); + } + // A batch naming a key twice lands the last, in the single-file and the redb stores alike. + for path in [temp.path().join("dup.cc"), temp.path().join("dup.db")] { + let mut cache = CultCache::new(); + cache.register_registry(TestEntries)?; + if path.extension().is_some_and(|extension| extension == "db") { + cache.add_generic_backing_store(RedbMessagePackBackingStore::new(&path)?)?; + } else { + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?; + } + cache.pull_all_backing_stores()?; + let (first, _) = cache.prepare_entry("k", &Settings { theme: "first".to_string(), retries: 1 })?; + let (last, _) = cache.prepare_entry("k", &Settings { theme: "last".to_string(), retries: 2 })?; + cache.put_prepared_batch(vec![first, last])?; + assert_eq!(cache.get_required::("k")?.theme, "last"); + } + Ok(()) + } + + #[test] + fn push_all_replaces_a_record_held_under_another_schema_id_as_push_does() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut store = SingleFileMessagePackBackingStore::new(temp.path().join("store.cc")); + let mut held = snapshot_envelope("k", b"old"); + held.schema_id = Some("legacy.id".to_string()); + store.push(&held)?; + let mut replacement = snapshot_envelope("k", b"new"); + replacement.schema_id = None; + let mut by_push = store.clone(); + by_push.push(&replacement)?; + assert_eq!(store.pull_all()?.len(), 1); + store.push(&held)?; + store.push_all(&[replacement.clone()], PushAllOptions::default())?; + let held_now = store.pull_all()?; + assert_eq!(held_now.len(), 1, "one record under the key"); + assert_eq!(held_now[0].payload, replacement.payload); + Ok(()) + } } From b098d5b7da5a884fc1af5488aaccdc5c1931812f Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 09:47:33 +0000 Subject: [PATCH 11/15] TypeScript: pushAll writes onto the store instead of replacing it (ts-flush-replaces-store) (#136) --- packages/cultcache-ts/CHANGELOG.md | 1 + packages/cultcache-ts/README.md | 2 +- packages/cultcache-ts/src/cult-cache.ts | 7 +++++ .../single-file-messagepack-backing-store.ts | 22 +++++++------ packages/cultcache-ts/test/cult-cache.test.ts | 31 ++++++++++++++++--- packages/cultcache-ts/test/store-file.test.ts | 11 +++++-- packages/cultmesh-ts/src/index.ts | 2 +- packages/cultmesh-ts/test/cultmesh.test.ts | 30 +++++++++++++++++- 8 files changed, 88 insertions(+), 18 deletions(-) diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 2da49fe9..dcbfd12e 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Breaking +- `SingleFileMessagePackBackingStore.pushAll` writes the given entries onto the store as it is under the lock, as `push` does: a record under the same key is replaced and no other is removed. It used to replace the whole file with the entries, so `CultMesh.flush()` from a cache that held less than the store (a node created with `pullOnStart: false`, or one whose image was stale) wiped the records it never read. A caller that relied on `pushAll` to reset a store removes the file first. A soft flush and the dangling-link case are unchanged. `CultCache.storedSnapshot()` is the image as the stores hold it (a global loaded from a legacy key under that key) and is what `CultMesh.flush` persists, so a flush no longer leaves such a global under two keys; `snapshot()` is unchanged. - A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index a081cd92..625d28dd 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -174,7 +174,7 @@ const settings = cache.getRequiredGlobal(settingsDocument); - `updateGlobal(...)` - `delete(...)` - `deleteGlobal(...)` -- `snapshot()` +- `snapshot()` (the logical image) and `storedSnapshot()` (the image as the stores hold it, for a whole flush) - `getEnvelope(...)` - `getRequiredEnvelope(...)` - `getGlobalEnvelope(...)` diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 6cf7a8cb..a88bd66d 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -636,6 +636,13 @@ export class CultCache { return [...this.#hydrated.entries.values()].map((entry) => this.#toEnvelope(entry)); } + // The image as the stores hold it, for persisting it whole (a flush): a global loaded from a legacy key is handed out under + // that key until its first write retires it, so a flush cannot leave the store holding it under two keys. `snapshot` is the + // logical image and always names a global by CultCache.GLOBAL_KEY. + storedSnapshot(): CultCacheEnvelope[] { + return [...this.#hydrated.entries.values()].map((entry) => this.#hydrated.legacyGlobals.get(entry.type) ?? this.#toEnvelope(entry)); + } + #registerNow(definition: AnyCultCacheDocumentDefinition): void { // Every registration refusal is a SchemaConflictError naming the new definition's schema id and both schema names. const refuse = (holder: RegisteredDefinition, claimed: CultCacheSchemaCatalogEntry, what: string) => diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 5dfcfc74..298f1bee 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -82,10 +82,12 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { async pushAll(entries: CultCacheEnvelope[], options: PushAllOptions = {}): Promise { await this.#enqueue(async () => { - // A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids - // stays marked, and one that is not (or is gone or legacy) is written unmarked. The file is read by the same - // reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused - // and left as it is. A soft flush asks that reader too: nothing at the path is written, a store is left alone. + // A flush writes `entries` onto the store as it is under the lock, the way a push does: a record under the same key + // is replaced and no other is removed, so a cache whose image is stale (or empty, never pulled) cannot drop what + // another writer added. The store is read by the same reader as `pullAll`, so one it would refuse (not exactly one + // store, a variant, a body it cannot decode) is refused and left as it is. A soft flush asks that reader too: nothing + // at the path is written, a store is left alone. + let held: CultCacheEnvelope[] = []; try { const disk = this.#readDisk(); if (options.soft && !disk.absent) { @@ -93,10 +95,11 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } this.#format = disk.format; + held = disk.envelopes; } catch (error) { - // Only a dangling link reaches here with ENOENT: it holds no store, so a whole-store flush has no header to keep and - // replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every merging writer - // has already refused the link when it read the current entries. + // Only a dangling link reaches here with ENOENT: it holds no store, so a flush has nothing to merge with and no + // header to keep, and replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every + // merging writer has already refused the link when it read the current entries. if (options.soft || (error as NodeJS.ErrnoException).code !== "ENOENT") { throw error; } @@ -104,9 +107,10 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { this.#format = STORE_FORMAT_VERSION; } - requireOneTypePerKey(this.filePath, entries, []); // A key is held once: of same-type entries sharing a key in one batch, the last is kept. - await this.#writeAll([...new Map(entries.map((entry) => [entry.key, entry])).values()], []); + const supplied = [...new Map(entries.map((entry) => [entry.key, entry])).values()]; + requireOneTypePerKey(this.filePath, entries, held); + await this.#writeAll(supplied, held); }); } diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index b575170f..2636a9d3 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -1019,6 +1019,7 @@ test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async () assert.deepEqual(reloaded.getRequiredGlobal(settingsDocument), { t: "new" }); // Deleting an adopted global removes the legacy record. + await rm(storePath); await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy]); const deleting = build(); await deleting.pullAllBackingStores(); @@ -1026,10 +1027,30 @@ test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async () assert.deepEqual(await new SingleFileMessagePackBackingStore(storePath).pullAll(), []); // Two globals of one type on disk, under any keys, are refused. + await rm(storePath); await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy, { ...legacy, key: CultCache.GLOBAL_KEY }]); await assert.rejects(build().pullAllBackingStores(), /has multiple persisted entries/u); }); +test("a flush of a cache that loaded a legacy-key global leaves the store reopenable", async () => { + const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ t: z.string() }), global: true }); + const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-legacy-flush-")), "generic.cc"); + const store = new SingleFileMessagePackBackingStore(storePath); + const build = () => CultCache.builder().withRegistry(defineDocumentRegistry(settingsDocument)).withGenericStore(store).build(); + const seed = CultCache.builder().withRegistry(defineDocumentRegistry(settingsDocument)).build(); + await seed.putGlobal(settingsDocument, { t: "old" }); + await store.push({ ...seed.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY), key: "legacy" }); + + const cache = build(); + await cache.pullAllBackingStores(); + assert.equal(cache.snapshot()[0]!.key, CultCache.GLOBAL_KEY, "the logical image names the global by the global key"); + await store.pushAll(cache.storedSnapshot()); + assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["legacy"], "the flush wrote the record under the key the store holds it"); + const reopened = build(); + await reopened.pullAllBackingStores(); + assert.deepEqual(reopened.getRequiredGlobal(settingsDocument), { t: "old" }); +}); + test("concurrentPutAndAttachCannotLandRecordInNonHomeStore", async () => { const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ t: z.string() }) }); const tempDir = await mkdtemp(join(tmpdir(), "cultcache-put-attach-race-")); @@ -2261,7 +2282,7 @@ test("a stamp this runtime reads, followed by bytes that are not UTF-8, gets the } }); -test("pushAll leaves a store it reads alone when soft, and replaces one it reads otherwise", async () => { +test("pushAll leaves a store it reads alone when soft, and writes onto one it reads otherwise", async () => { const vector = await readFile(resolve(cultLibRoot, "tests", "vectors", "document-variants-c2a", "v3-base.msgpack")); const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; const existing = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-soft-")), "store.msgpack"); @@ -2273,7 +2294,7 @@ test("pushAll leaves a store it reads alone when soft, and replaces one it reads const store = new SingleFileMessagePackBackingStore(own); await store.pushAll([envelope]); await store.pushAll([{ ...envelope, key: "second" }]); - assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["second"]); + assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["probe", "second"]); }); test("SingleFileMessagePackBackingStore reads only a missing store as empty", async () => { @@ -2575,7 +2596,7 @@ test("SingleFileMessagePackBackingStore pushAll refuses a store whose header it // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by pullAll, // pushAll, soft pushAll, push and delete alike; a soft flush leaves the files that read as they are. The bytes and every runtime's verdict: tests/vectors/document-variants-c2a/readability. -test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", async () => { +test("SingleFileMessagePackBackingStore rewrites a file exactly when it reads", async () => { const root = join(c2aVectors, "readability"); const rows = (await readFile(join(root, "manifest.txt"), "utf8")) .split(/\r?\n/u) @@ -2612,6 +2633,7 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", ? store.push(envelopes[0]!) : store.delete(envelopes[0]!); if (reads) { + const heldBefore = operation === "pushAll" ? (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key) : []; await run(); if (operation === "pullAll") { continue; @@ -2628,7 +2650,8 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", assert.equal((decode(await readFile(file)) as unknown[])[0], vector.includes("v3") ? "cultcache.store.v3" : "cultcache.store.v1", `${vector} ${operation}`); const keys = (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key); if (operation === "pushAll") { - assert.deepEqual(keys, envelopes.map((entry) => entry.key).sort(), `${vector} ${operation}`); + assert.deepEqual(envelopes.map((entry) => entry.key).filter((key) => !keys.includes(key)), [], `${vector} ${operation}: every written key is held`); + assert.deepEqual(heldBefore.filter((key) => !keys.includes(key)), [], `${vector} ${operation}: nothing else is removed`); } else if (operation === "push") { assert.ok(keys.includes(envelopes[0]!.key), `${vector} ${operation}`); } else { diff --git a/packages/cultcache-ts/test/store-file.test.ts b/packages/cultcache-ts/test/store-file.test.ts index eb41b77b..39436e30 100644 --- a/packages/cultcache-ts/test/store-file.test.ts +++ b/packages/cultcache-ts/test/store-file.test.ts @@ -670,7 +670,7 @@ test("delete removes only the record at that type and key", async () => { } }); -test("pushAll replaces the whole store, and a soft pushAll leaves an existing store alone", async () => { +test("pushAll writes onto the store as it is: it replaces by key and removes nothing, and a soft pushAll leaves an existing store alone", async () => { const directory = await scratch(); try { const path = join(directory, "store.cc"); @@ -681,7 +681,14 @@ test("pushAll replaces the whole store, and a soft pushAll leaves an existing st await store.pushAll([entryOf("b", "t", 2)], { soft: true }); assert.deepEqual([...(await readFile(path))], [...before], "soft leaves an existing store byte for byte"); await store.pushAll([entryOf("b", "t", 2)]); - assert.deepEqual(await snapshotOf(store), ["t/b:2"]); + assert.deepEqual(await snapshotOf(store), ["t/a:1", "t/b:2"], "a record the batch does not name stays"); + await store.pushAll([entryOf("a", "t", 7)]); + assert.deepEqual(await snapshotOf(store), ["t/a:7", "t/b:2"], "a record the batch names is replaced"); + await store.pushAll([]); + assert.deepEqual(await snapshotOf(store), ["t/a:7", "t/b:2"], "an empty flush from a cache that holds nothing wipes nothing"); + const before2 = [...(await readFile(path))]; + await assert.rejects(() => store.pushAll([entryOf("a", "u", 1)]), SchemaConflictError); + assert.deepEqual([...(await readFile(path))], before2, "a batch that retypes a held key is refused and the file is unchanged"); } finally { await rm(directory, { recursive: true, force: true }); } diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index 2f734cfd..e6f71e21 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -3933,7 +3933,7 @@ export class CultMeshNode { } public async flush(soft = false): Promise { - await this.store.pushAll?.(this.cache.snapshot(), { soft }); + await this.store.pushAll?.(this.cache.storedSnapshot(), { soft }); } } diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 10de21b9..341a35b3 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -6,7 +6,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire"; import { z } from "zod"; -import { SchemaConflictError, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; +import { SchemaConflictError, SingleFileMessagePackBackingStore, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts"; import { CultNetDocumentRegistry, CultNetPeer, @@ -109,6 +109,34 @@ test("CultMesh TS opens a durable local node and registers document bindings", a assert.ok(reopened.documents.getBySchemaId("cultmesh.note.v0")); }); +test("CultMesh TS flush from a node that never pulled leaves the records the store holds", async () => { + const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-flush-")), "node.ccmp"); + const first = await CultMesh.startNode(filePath, { documents: [noteDocument] }); + await first.put(noteDocument, "note:held", { noteId: "note:held", body: "already in the store" }); + await first.flush(); + + const unpulled = await CultMesh.startNode(filePath, { documents: [noteDocument], pullOnStart: false }); + await unpulled.flush(); + + const reopened = await CultMesh.startNode(filePath, { documents: [noteDocument] }); + assert.equal(reopened.getRequired(noteDocument, "note:held").body, "already in the store"); +}); + +test("CultMesh TS flush of a node that loaded a legacy-key global leaves the store reopenable", async () => { + const settings = defineDocumentType({ type: "cultmesh.settings", schema: z.object({ theme: z.string() }), global: true }); + const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-legacy-flush-")), "node.ccmp"); + const seed = await CultMesh.startNode(join(await mkdtemp(join(tmpdir(), "cultmesh-ts-seed-")), "seed.ccmp"), { documents: [settings] }); + await seed.cache.putGlobal(settings, { theme: "iron" }); + const legacy = { ...seed.cache.getRequiredEnvelope(settings, "__global__"), key: "legacy" }; + await new SingleFileMessagePackBackingStore(filePath).push(legacy); + + const node = await CultMesh.startNode(filePath, { documents: [settings] }); + await node.flush(); + + const reopened = await CultMesh.startNode(filePath, { documents: [settings] }); + assert.deepEqual(reopened.cache.getRequiredGlobal(settings), { theme: "iron" }); +}); + test("CultMesh TS document handles hide local cache plumbing behind typed reactive reads", async () => { const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-doc-")), "node.ccmp"); const node = await CultMesh.startNode(filePath, { From 2fd80c274b5f8691d360297f5cf95f2aeb2488e8 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 09:47:38 +0000 Subject: [PATCH 12/15] Rust mesh: a raw snapshot pull is one cache commit (mesh-snapshot-atomic) (#134) --- packages/cultcache-rs/CHANGELOG.md | 2 +- packages/cultmesh-rs/src/lib.rs | 40 ++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index b43c13c6..e1fad67d 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -11,7 +11,7 @@ only in git history. Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0). - A record whose payload carries unread slots is read-only. A payload with a slot past its type's declared slots, at any depth (record, nested record, list element, union arm, unit arm), still decodes, and every write or removal of its key is now refused before any store is touched: `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` (the whole batch) and `delete`, so `update` too. The refusal is `SchemaConflictError` with the new `unread_slots`; its message names the type and both slot counts and never a slot value. Holders of such a record used to overwrite it and drop the extra slots silently. The payload nesting refusal still runs first. A held payload this build cannot read at all (an arm it lacks from a newer writer, or a corrupt record) is read-only too: every write or removal is refused with an error naming the type and key (a newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, as no cache call removes it), and `is_writable` answers false. `load_soa` is a write of every held key it omits or changes, so it refuses to drop or replace a read-only record. `is_writable` also answers false for a key a store carries as a foreign record and for a type other than the one holding the key, as the writes do. Each write over a held key decodes the held payload once to ask (about 40% of a `put` of a 200,000-element record, measured in release); keys nothing holds pay nothing. -- cultmesh-rs `pull_rudp_catalog_snapshot` applies documents one by one with `put_raw_envelope`, so a snapshot document for a key the node holds read-only ends the pull at that document and the documents before it stay applied. C# applies a raw snapshot as one cache commit, so there a refusal applies nothing: this is a divergence, owed by follow-up `mesh-snapshot-atomic` (stage the pull through `put_prepared_batch`). +- cultmesh-rs `pull_rudp_catalog_snapshot` applies a snapshot as one cache commit (`put_prepared_batch`), as C# does: a snapshot document for a key the node holds read-only, or any other refusal, applies nothing. It used to apply document by document, so a refusal ended the pull with the earlier documents kept. A snapshot whose types route to different backing stores is now refused whole, as a batch cannot span stores. - `SchemaConflictError` gains the public field `unread_slots: Option`; code that builds it with a struct literal must set it (`None`). - A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it. - The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check. diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 18a02fcb..310629e3 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -735,30 +735,33 @@ impl CultMeshNode { self.apply_raw_snapshot_documents(documents) } - /// Applies each snapshot document of a registered schema with `put_raw_envelope`. The first refusal ends the pull - /// and the documents before it stay applied; a key this node holds read-only (unread slots from a newer peer, or a - /// payload this build cannot read) is refused like any other. C# applies a raw snapshot as one cache commit, so a - /// refusal there applies nothing: this is a divergence, owed by follow-up mesh-snapshot-atomic (stage the pull through - /// `put_prepared_batch`). + /// Applies the snapshot documents as one cache commit, as C# does: a refusal (a key this node holds read-only because + /// of unread slots from a newer peer or a payload this build cannot read, or any other the cache judges) applies + /// nothing. A schema id the registry does not know is skipped, except one with a `.vN` suffix, which is taken as + /// registered and refuses the commit when its type is not (C# throws on an unknown schema too). The commit lands in + /// one backing store, so a snapshot spanning stores is refused whole. fn apply_raw_snapshot_documents( &mut self, documents: Vec, ) -> Result { - let mut applied = 0usize; + let mut staged = Vec::new(); for document in documents { let Some(document_type) = registered_document_type_for_schema(&self.documents, &document.schema_id) else { continue; }; - self.cache.put_raw_envelope(CultCacheEnvelope { + staged.push(CultCacheEnvelope { key: document.record_key, r#type: document_type, payload: document.payload, stored_at: document.stored_at, schema_id: Some(document.schema_id), - })?; - applied += 1; + }); + } + let applied = staged.len(); + if applied > 0 { + self.cache.put_prepared_batch(staged)?; } Ok(applied) } @@ -1226,7 +1229,7 @@ mod tests { } #[test] - fn a_snapshot_document_for_a_read_only_key_ends_the_pull_unlike_csharp_which_applies_nothing() -> Result<()> { + fn a_snapshot_document_for_a_read_only_key_refuses_the_whole_pull_as_csharp_does() -> Result<()> { let temp = tempfile::tempdir()?; let mut node = CultMesh::create_node(temp.path().join("m.cc"), TestDocuments, Default::default())?; // A newer peer's note: one slot past the declared one. @@ -1246,12 +1249,27 @@ mod tests { assert_eq!(conflict.record_key, "held"); assert!(conflict.unread_slots.is_some()); assert!(!error.to_string().contains("extra slot")); - assert_eq!(node.get_required::("before")?.body, "b"); + assert!(node.get::("before")?.is_none()); assert_eq!(node.get_required::("held")?.body, "newer body"); assert!(node.get::("after")?.is_none()); Ok(()) } + #[test] + fn a_snapshot_applied_to_a_stale_node_keeps_the_records_another_writer_added() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("m.cc"); + let mut node = CultMesh::create_node(&path, TestDocuments, Default::default())?; + let mut other = CultMesh::create_node(&path, TestDocuments, Default::default())?; + other.put("added", &Note { body: "other writer".to_string() })?; + let plain = |body: &str| rmp_serde::to_vec(&(body,)); + assert_eq!(node.apply_raw_snapshot_documents(vec![raw_note("snap", plain("s")?)])?, 1); + let reopened = CultMesh::create_node(&path, TestDocuments, Default::default())?; + assert_eq!(reopened.get_required::("added")?.body, "other writer"); + assert_eq!(reopened.get_required::("snap")?.body, "s"); + Ok(()) + } + #[test] fn node_round_trips_registered_documents() -> Result<()> { let temp = tempfile::tempdir()?; From a23ac7755acff318b134300b7003d2383d9f28c3 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 10:31:13 +0000 Subject: [PATCH 13/15] Rust mesh: pin that a snapshot spanning backing stores is refused whole (#137) --- packages/cultcache-rs/CHANGELOG.md | 2 +- packages/cultmesh-rs/src/lib.rs | 48 +++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index e1fad67d..7f3caa73 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -11,7 +11,7 @@ only in git history. Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0). - A record whose payload carries unread slots is read-only. A payload with a slot past its type's declared slots, at any depth (record, nested record, list element, union arm, unit arm), still decodes, and every write or removal of its key is now refused before any store is touched: `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` (the whole batch) and `delete`, so `update` too. The refusal is `SchemaConflictError` with the new `unread_slots`; its message names the type and both slot counts and never a slot value. Holders of such a record used to overwrite it and drop the extra slots silently. The payload nesting refusal still runs first. A held payload this build cannot read at all (an arm it lacks from a newer writer, or a corrupt record) is read-only too: every write or removal is refused with an error naming the type and key (a newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, as no cache call removes it), and `is_writable` answers false. `load_soa` is a write of every held key it omits or changes, so it refuses to drop or replace a read-only record. `is_writable` also answers false for a key a store carries as a foreign record and for a type other than the one holding the key, as the writes do. Each write over a held key decodes the held payload once to ask (about 40% of a `put` of a 200,000-element record, measured in release); keys nothing holds pay nothing. -- cultmesh-rs `pull_rudp_catalog_snapshot` applies a snapshot as one cache commit (`put_prepared_batch`), as C# does: a snapshot document for a key the node holds read-only, or any other refusal, applies nothing. It used to apply document by document, so a refusal ended the pull with the earlier documents kept. A snapshot whose types route to different backing stores is now refused whole, as a batch cannot span stores. +- cultmesh-rs `pull_rudp_catalog_snapshot` applies a snapshot as one cache commit (`put_prepared_batch`), as C# does: a snapshot document for a key the node holds read-only, or any other refusal, applies nothing. It used to apply document by document, so a refusal ended the pull with the earlier documents kept. A snapshot whose types route to different backing stores is now refused whole, as a batch cannot span stores. C# `CultCache.Commit` refuses a cross-store commit the same way (`The batch spans A and B; a commit lands in one home store`), so the two agree; a test pins it. - `SchemaConflictError` gains the public field `unread_slots: Option`; code that builds it with a struct literal must set it (`None`). - A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it. - The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check. diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 310629e3..b1ee613b 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -739,7 +739,8 @@ impl CultMeshNode { /// of unread slots from a newer peer or a payload this build cannot read, or any other the cache judges) applies /// nothing. A schema id the registry does not know is skipped, except one with a `.vN` suffix, which is taken as /// registered and refuses the commit when its type is not (C# throws on an unknown schema too). The commit lands in - /// one backing store, so a snapshot spanning stores is refused whole. + /// one backing store, so a snapshot spanning stores is refused whole; C# `CultCache.Commit` refuses a cross-store batch too + /// ("The batch spans A and B; a commit lands in one home store"), so the two agree. fn apply_raw_snapshot_documents( &mut self, documents: Vec, @@ -1210,10 +1211,22 @@ mod tests { body: String, } + #[derive(Clone, Debug, PartialEq, Eq, cultcache_rs::DatabaseEntry)] + #[cultcache(type = "cultmesh.test.memo", schema = "CultMeshTestMemo")] + struct Memo { + #[cultcache(key = 0)] + body: String, + } + cultmesh_documents!(TestDocuments { Note => "cultmesh.test.note.v0", }); + cultmesh_documents!(TwoTypeDocuments { + Note => "cultmesh.test.note.v0", + Memo => "cultmesh.test.memo.v0", + }); + fn raw_note(key: &str, payload: Vec) -> cultnet_rs::CultNetRawDocumentRecord { cultnet_rs::CultNetRawDocumentRecord { schema_id: "cultmesh.test.note.v0".to_string(), @@ -1255,6 +1268,39 @@ mod tests { Ok(()) } + #[test] + fn a_snapshot_spanning_two_backing_stores_is_refused_whole_and_applies_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut node = CultMesh::create_node(temp.path().join("m.cc"), TwoTypeDocuments, Default::default())?; + // A node has one store through the public API; route Memo to a second one by hand (C# refuses the same commit). + node.cache.add_backing_store( + SingleFileMessagePackBackingStore::new(temp.path().join("memo.cc")), + [Memo::TYPE], + )?; + node.cache.pull_all_backing_stores()?; + let plain = |body: &str| rmp_serde::to_vec(&(body,)); + let memo = |key: &str, payload| { + let mut record = raw_note(key, payload); + record.schema_id = "cultmesh.test.memo.v0".to_string(); + record + }; + let error = node + .apply_raw_snapshot_documents(vec![ + raw_note("n", plain("note")?), + memo("m", plain("memo")?), + ]) + .unwrap_err(); + assert!(error.to_string().contains("cannot span backing stores"), "{error}"); + assert!(node.get::("n")?.is_none()); + assert!(node.get::("m")?.is_none()); + assert!(!temp.path().join("m.cc").exists() && !temp.path().join("memo.cc").exists()); + + // Each store alone still takes its own snapshot. + assert_eq!(node.apply_raw_snapshot_documents(vec![memo("m", plain("memo")?)])?, 1); + assert_eq!(node.get_required::("m")?.body, "memo"); + Ok(()) + } + #[test] fn a_snapshot_applied_to_a_stale_node_keeps_the_records_another_writer_added() -> Result<()> { let temp = tempfile::tempdir()?; From f4fe424d4154e34d978b4d47e085b8538ceeefbb Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 12:16:52 +0000 Subject: [PATCH 14/15] CultCache interop: C# union and nested-bytes store read by Rust read-only, and back (#139) --- .../examples/cultcache_interop.rs | 98 +++++++++++++++++++ packages/cultcache-ts/test/cult-cache.test.ts | 45 ++++++++- tests/GameCult.Caching.InteropPeer/Program.cs | 74 ++++++++++++++ 3 files changed, 215 insertions(+), 2 deletions(-) diff --git a/packages/cultcache-rs/examples/cultcache_interop.rs b/packages/cultcache-rs/examples/cultcache_interop.rs index 8efb9395..dcafee5f 100644 --- a/packages/cultcache-rs/examples/cultcache_interop.rs +++ b/packages/cultcache-rs/examples/cultcache_interop.rs @@ -1,6 +1,8 @@ use anyhow::Result; use anyhow::anyhow; use cultcache_rs::CultCache; +use cultcache_rs::CultSlots; +use cultcache_rs::CultUnion; use cultcache_rs::DatabaseEntry; use cultcache_rs::SingleFileMessagePackBackingStore; use serde_json::json; @@ -39,6 +41,100 @@ struct CultCacheInteropDeck { marks: Vec, } +// A union field and nested bytes: the shapes a Bifrost-style store holds beyond plain strings. +#[derive(Clone, Debug, PartialEq, Eq, CultUnion)] +enum InteropArm { + #[cultcache(key = 0)] + Text(InteropTextArm), + #[cultcache(key = 1)] + Pack(InteropPackArm), +} + +#[derive(Clone, Debug, PartialEq, Eq, CultSlots)] +struct InteropTextArm { + #[cultcache(key = 0)] + text: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, CultSlots)] +struct InteropPackArm { + #[cultcache(key = 0)] + label: String, + #[cultcache(key = 1)] + chunks: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "cultcache.interop-shape", schema = "CultCacheInteropShape")] +struct CultCacheInteropShape { + #[cultcache(key = 0)] + document_id: String, + #[cultcache(key = 1)] + arm: InteropArm, + #[cultcache(key = 2)] + raw: serde_bytes::ByteBuf, +} + +fn shapes() -> Vec { + vec![ + CultCacheInteropShape { + document_id: "shape:text".into(), + arm: InteropArm::Text(InteropTextArm { text: "union arm 0".into() }), + raw: (0..=255u8).collect::>().into(), + }, + CultCacheInteropShape { + document_id: "shape:pack".into(), + arm: InteropArm::Pack(InteropPackArm { + label: "union arm 1".into(), + chunks: vec![vec![0xc0, 0xc1, 0x00].into(), vec![].into(), vec![0x92, 0x01, 0x02].into()], + }), + raw: vec![0xde, 0xad, 0xbe, 0xef].into(), + }, + ] +} + +fn write_shapes(file: &str) -> Result<()> { + let mut cache = CultCache::new(); + cache.register_entry_type::()?; + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file))?; + cache.pull_all_backing_stores()?; + for shape in shapes() { + cache.put(&shape.document_id, &shape)?; + } + print_shapes(shapes()) +} + +/// Opens the store through the read-only snapshot door: no lock file, no staging file, no rewrite. +fn read_shapes(file: &str) -> Result<()> { + let envelopes = SingleFileMessagePackBackingStore::new(file).pull_all_read_only_snapshot()?; + let mut found = Vec::new(); + for envelope in envelopes.iter().filter(|envelope| envelope.r#type == CultCacheInteropShape::TYPE) { + found.push(cultcache_rs::decode_payload::(&envelope.payload)?); + } + print_shapes(found) +} + +fn print_shapes(mut found: Vec) -> Result<()> { + let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02X}")).collect::(); + found.sort_by(|a, b| a.document_id.cmp(&b.document_id)); + let rows: Vec<_> = found + .iter() + .map(|shape| { + let arm = match &shape.arm { + InteropArm::Text(text) => json!({ "kind": "text", "text": text.text }), + InteropArm::Pack(pack) => json!({ + "kind": "pack", + "label": pack.label, + "chunks": pack.chunks.iter().map(|chunk| hex(chunk)).collect::>(), + }), + }; + json!({ "documentId": shape.document_id, "arm": arm, "raw": hex(&shape.raw) }) + }) + .collect(); + println!("{}", serde_json::to_string(&rows)?); + Ok(()) +} + fn read_deck(file: &str) -> Result<()> { let mut cache = CultCache::new(); cache.register_entry_type::()?; @@ -70,6 +166,8 @@ fn main() -> Result<()> { "write" => write_note(file, require_arg(&options, "runtime-id")?)?, "read" => read_note(file)?, "read-deck" => read_deck(file)?, + "write-shapes" => write_shapes(file)?, + "read-shapes" => read_shapes(file)?, "hold-lock" => hold_lock(file, require_arg(&options, "ms")?.parse()?)?, _ => return Err(anyhow!("unknown mode {mode}")), } diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 2636a9d3..4efa6d78 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -33,8 +33,7 @@ const cultcacheRsRoot = existsSync(resolve(cultLibRoot, "packages", "cultcache-r ? resolve(cultLibRoot, "packages", "cultcache-rs") : resolve(cultCacheTsRoot, "..", "cultcache-rs"); const rustInteropBinary = resolve( - cultcacheRsRoot, - "target", + process.env.CARGO_TARGET_DIR ?? resolve(cultcacheRsRoot, "target"), "debug", "examples", process.platform === "win32" ? "cultcache_interop.exe" : "cultcache_interop", @@ -1273,6 +1272,48 @@ test("CultCache v1 MessagePack stores are readable across TS, Rust, C#, and Pyth } }); +// A Bifrost-shaped store (a union field, nested bytes) written by one of C# and Rust is opened by the other: C# reads through +// its store, Rust through the read-only snapshot door, which takes no lock and writes nothing. Leftover staging files from an +// interrupted write of either runtime sit beside the store; a read neither deletes them nor lets them change what it reads, and +// the store file's bytes are the same after every read. +test("A store written by C# or Rust with a union and nested bytes is read back equal by the other, read-only, past leftover staging files", async () => { + await buildInteropPeers(); + const tempDir = await mkdtemp(join(tmpdir(), "cultcache-shapes-")); + const rustRead = (file: string) => runJsonCommand("rust-read-shapes", rustInteropBinary, ["read-shapes", "--file", file], cultcacheRsRoot); + const csharpRead = (file: string) => runJsonCommand("csharp-read-shapes", dotnetCommand, [csharpInteropDll, "read-shapes", "--file", file], cultLibRoot); + const peers = [ + { + writer: "csharp", + write: (file: string) => runJsonCommand("csharp-write-shapes", dotnetCommand, [csharpInteropDll, "write-shapes", "--file", file], cultLibRoot), + read: rustRead, + }, + { + writer: "rust", + write: (file: string) => runJsonCommand("rust-write-shapes", rustInteropBinary, ["write-shapes", "--file", file], cultcacheRsRoot), + read: csharpRead, + }, + ]; + for (const { writer, write, read } of peers) { + const file = join(tempDir, `${writer}.cc`); + const written = await write(file); + assert.equal(written.length, 2); + // The leftovers an interrupted write leaves: C# stages as `...tmp`, Rust as `..tmp`; both hold partial bytes. + const leftovers = [ + join(tempDir, `.${writer}.cc.0123456789abcdef0123456789abcdef.tmp`), + join(tempDir, `${writer}.cc.00000000-0000-4000-8000-000000000000.tmp`), + ]; + for (const leftover of leftovers) { + await writeFile(leftover, Buffer.from([0x93, 0xb2])); + } + const before = await readFile(file); + assert.deepEqual(await read(file), written, `${writer}'s store did not read back equal`); + assert.deepEqual(await readFile(file), before, `reading ${writer}'s store changed its bytes`); + for (const leftover of leftovers) { + assert.ok(await exists(leftover), `reading ${writer}'s store removed ${leftover}`); + } + } +}); + // Records and catalog entries a runtime does not own survive its writes. C# writes a store and each other runtime writes its own // note into it: C# reads its note back, and its record and catalog entry are as C# wrote them. C# then writes again, and the other // runtime's record and catalog entry are as that runtime wrote them. Identity is compared on the decoded MessagePack values, every diff --git a/tests/GameCult.Caching.InteropPeer/Program.cs b/tests/GameCult.Caching.InteropPeer/Program.cs index 3c8eb92b..f0f0479e 100644 --- a/tests/GameCult.Caching.InteropPeer/Program.cs +++ b/tests/GameCult.Caching.InteropPeer/Program.cs @@ -51,6 +51,12 @@ static async Task ProgramMainAsync(string[] args) case "edit-in-place": await EditInPlaceAsync(file, RequireArg(options, "kind")); return 0; + case "write-shapes": + await WriteShapesAsync(file); + return 0; + case "read-shapes": + await ReadShapesAsync(file); + return 0; case "hold-lock": HoldLock(file, int.Parse(RequireArg(options, "ms"), CultureInfo.InvariantCulture)); return 0; @@ -96,6 +102,49 @@ static async Task WriteAsync(string file, string runtimeId) WriteJsonLine(note); } +// Records a reader of a Bifrost-shaped store must open: a union field (two arms) and nested bytes (a bin member, a bin list). +static async Task WriteShapesAsync(string file) +{ + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + foreach (var shape in InteropShapes()) + await cache.AddAsync(shape, new CultRecordHandle(new CultRecordKey(shape.DocumentId))); + cache.FlushAllBackingStores(); + WriteShapesJsonLine(InteropShapes()); +} + +static async Task ReadShapesAsync(string file) +{ + var cache = BuildCache(file); + await cache.PullAllBackingStoresAsync(); + WriteShapesJsonLine(cache.AllEntries.OfType()); +} + +static CultCacheInteropShape[] InteropShapes() => +[ + new() { DocumentId = "shape:text", Arm = new InteropTextArm { Text = "union arm 0" }, Raw = Enumerable.Range(0, 256).Select(i => (byte)i).ToArray() }, + new() + { + DocumentId = "shape:pack", + Arm = new InteropPackArm { Label = "union arm 1", Chunks = [new byte[] { 0xc0, 0xc1, 0x00 }, [], [0x92, 0x01, 0x02]] }, + Raw = [0xde, 0xad, 0xbe, 0xef] + } +]; + +// One line, ordered by document id, in the form the Rust peer prints. +static void WriteShapesJsonLine(IEnumerable shapes) => + Console.Out.WriteLine(JsonSerializer.Serialize(shapes.OrderBy(shape => shape.DocumentId, StringComparer.Ordinal).Select(shape => new + { + documentId = shape.DocumentId, + arm = shape.Arm switch + { + InteropTextArm text => (object)new { kind = "text", text = text.Text }, + InteropPackArm pack => new { kind = "pack", label = pack.Label, chunks = pack.Chunks.Select(Convert.ToHexString).ToArray() }, + _ => throw new InvalidOperationException("Unknown arm.") + }, + raw = Convert.ToHexString(shape.Raw) + }))); + static async Task ReadAsync(string file) { var cache = BuildCache(file); @@ -328,3 +377,28 @@ public sealed class CultCacheInteropMark [Key(0)] public string Label { get; set; } = string.Empty; [Key(1)] [CultElementId] public string Id { get; set; } = string.Empty; } + +[CultDocument("cultcache.interop-shape", "cultcache.interop_shape.v1")] +[MessagePackObject] +public sealed class CultCacheInteropShape +{ + [Key(0)] [CultName] public string DocumentId { get; set; } = string.Empty; + [Key(1)] public InteropArm Arm { get; set; } = new InteropTextArm(); + [Key(2)] public byte[] Raw { get; set; } = Array.Empty(); +} + +[Union(0, typeof(InteropTextArm)), Union(1, typeof(InteropPackArm))] +public abstract class InteropArm { } + +[MessagePackObject] +public sealed class InteropTextArm : InteropArm +{ + [Key(0)] public string Text { get; set; } = string.Empty; +} + +[MessagePackObject] +public sealed class InteropPackArm : InteropArm +{ + [Key(0)] public string Label { get; set; } = string.Empty; + [Key(1)] public List Chunks { get; set; } = new(); +} From 5f5364e4dcea48df4e82b7a0061e97940de88ac6 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 12:16:58 +0000 Subject: [PATCH 15/15] Lower System.Text.Json floor to 8.0.5 (#140) --- .../GameCult.Caching.MessagePack.csproj | 2 +- src/GameCult.Geometry/GameCult.Geometry.csproj | 2 +- src/GameCult.Networking/GameCult.Networking.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj index 63cbf614..0afdd0d3 100644 --- a/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj +++ b/src/GameCult.Caching.MessagePack/GameCult.Caching.MessagePack.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/GameCult.Geometry/GameCult.Geometry.csproj b/src/GameCult.Geometry/GameCult.Geometry.csproj index 9736e1b7..744bd109 100644 --- a/src/GameCult.Geometry/GameCult.Geometry.csproj +++ b/src/GameCult.Geometry/GameCult.Geometry.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/GameCult.Networking/GameCult.Networking.csproj b/src/GameCult.Networking/GameCult.Networking.csproj index 08ef2ed6..68e13794 100644 --- a/src/GameCult.Networking/GameCult.Networking.csproj +++ b/src/GameCult.Networking/GameCult.Networking.csproj @@ -9,7 +9,7 @@ - +