diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 2da49fe9..2b226954 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Breaking +- `SingleFileMessagePackBackingStore.pushAll` writes the given entries onto the store as it is under the lock, as `push` does: a record under the same key is replaced and no other is removed. It used to replace the whole file with the entries, so `CultMesh.flush()` from a cache that held less than the store (a node created with `pullOnStart: false`, or one whose image was stale) wiped the records it never read. A caller that relied on `pushAll` to reset a store removes the file first. A soft flush and the dangling-link case are unchanged. - A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 5dfcfc74..298f1bee 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -82,10 +82,12 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { async pushAll(entries: CultCacheEnvelope[], options: PushAllOptions = {}): Promise { await this.#enqueue(async () => { - // A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids - // stays marked, and one that is not (or is gone or legacy) is written unmarked. The file is read by the same - // reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused - // and left as it is. A soft flush asks that reader too: nothing at the path is written, a store is left alone. + // A flush writes `entries` onto the store as it is under the lock, the way a push does: a record under the same key + // is replaced and no other is removed, so a cache whose image is stale (or empty, never pulled) cannot drop what + // another writer added. The store is read by the same reader as `pullAll`, so one it would refuse (not exactly one + // store, a variant, a body it cannot decode) is refused and left as it is. A soft flush asks that reader too: nothing + // at the path is written, a store is left alone. + let held: CultCacheEnvelope[] = []; try { const disk = this.#readDisk(); if (options.soft && !disk.absent) { @@ -93,10 +95,11 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } this.#format = disk.format; + held = disk.envelopes; } catch (error) { - // Only a dangling link reaches here with ENOENT: it holds no store, so a whole-store flush has no header to keep and - // replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every merging writer - // has already refused the link when it read the current entries. + // Only a dangling link reaches here with ENOENT: it holds no store, so a flush has nothing to merge with and no + // header to keep, and replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every + // merging writer has already refused the link when it read the current entries. if (options.soft || (error as NodeJS.ErrnoException).code !== "ENOENT") { throw error; } @@ -104,9 +107,10 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { this.#format = STORE_FORMAT_VERSION; } - requireOneTypePerKey(this.filePath, entries, []); // A key is held once: of same-type entries sharing a key in one batch, the last is kept. - await this.#writeAll([...new Map(entries.map((entry) => [entry.key, entry])).values()], []); + const supplied = [...new Map(entries.map((entry) => [entry.key, entry])).values()]; + requireOneTypePerKey(this.filePath, entries, held); + await this.#writeAll(supplied, held); }); } diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index b575170f..d2b515c9 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -1019,6 +1019,7 @@ test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async () assert.deepEqual(reloaded.getRequiredGlobal(settingsDocument), { t: "new" }); // Deleting an adopted global removes the legacy record. + await rm(storePath); await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy]); const deleting = build(); await deleting.pullAllBackingStores(); @@ -1026,6 +1027,7 @@ test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async () assert.deepEqual(await new SingleFileMessagePackBackingStore(storePath).pullAll(), []); // Two globals of one type on disk, under any keys, are refused. + await rm(storePath); await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy, { ...legacy, key: CultCache.GLOBAL_KEY }]); await assert.rejects(build().pullAllBackingStores(), /has multiple persisted entries/u); }); @@ -2261,7 +2263,7 @@ test("a stamp this runtime reads, followed by bytes that are not UTF-8, gets the } }); -test("pushAll leaves a store it reads alone when soft, and replaces one it reads otherwise", async () => { +test("pushAll leaves a store it reads alone when soft, and writes onto one it reads otherwise", async () => { const vector = await readFile(resolve(cultLibRoot, "tests", "vectors", "document-variants-c2a", "v3-base.msgpack")); const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" }; const existing = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-soft-")), "store.msgpack"); @@ -2273,7 +2275,7 @@ test("pushAll leaves a store it reads alone when soft, and replaces one it reads const store = new SingleFileMessagePackBackingStore(own); await store.pushAll([envelope]); await store.pushAll([{ ...envelope, key: "second" }]); - assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["second"]); + assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["probe", "second"]); }); test("SingleFileMessagePackBackingStore reads only a missing store as empty", async () => { @@ -2575,7 +2577,7 @@ test("SingleFileMessagePackBackingStore pushAll refuses a store whose header it // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by pullAll, // pushAll, soft pushAll, push and delete alike; a soft flush leaves the files that read as they are. The bytes and every runtime's verdict: tests/vectors/document-variants-c2a/readability. -test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", async () => { +test("SingleFileMessagePackBackingStore rewrites a file exactly when it reads", async () => { const root = join(c2aVectors, "readability"); const rows = (await readFile(join(root, "manifest.txt"), "utf8")) .split(/\r?\n/u) @@ -2628,7 +2630,8 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", assert.equal((decode(await readFile(file)) as unknown[])[0], vector.includes("v3") ? "cultcache.store.v3" : "cultcache.store.v1", `${vector} ${operation}`); const keys = (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key); if (operation === "pushAll") { - assert.deepEqual(keys, envelopes.map((entry) => entry.key).sort(), `${vector} ${operation}`); + assert.deepEqual(envelopes.map((entry) => entry.key).filter((key) => !keys.includes(key)), [], `${vector} ${operation}: every written key is held`); + assert.ok(keys.length >= envelopes.length, `${vector} ${operation}: nothing else is removed`); } else if (operation === "push") { assert.ok(keys.includes(envelopes[0]!.key), `${vector} ${operation}`); } else { diff --git a/packages/cultcache-ts/test/store-file.test.ts b/packages/cultcache-ts/test/store-file.test.ts index eb41b77b..39436e30 100644 --- a/packages/cultcache-ts/test/store-file.test.ts +++ b/packages/cultcache-ts/test/store-file.test.ts @@ -670,7 +670,7 @@ test("delete removes only the record at that type and key", async () => { } }); -test("pushAll replaces the whole store, and a soft pushAll leaves an existing store alone", async () => { +test("pushAll writes onto the store as it is: it replaces by key and removes nothing, and a soft pushAll leaves an existing store alone", async () => { const directory = await scratch(); try { const path = join(directory, "store.cc"); @@ -681,7 +681,14 @@ test("pushAll replaces the whole store, and a soft pushAll leaves an existing st await store.pushAll([entryOf("b", "t", 2)], { soft: true }); assert.deepEqual([...(await readFile(path))], [...before], "soft leaves an existing store byte for byte"); await store.pushAll([entryOf("b", "t", 2)]); - assert.deepEqual(await snapshotOf(store), ["t/b:2"]); + assert.deepEqual(await snapshotOf(store), ["t/a:1", "t/b:2"], "a record the batch does not name stays"); + await store.pushAll([entryOf("a", "t", 7)]); + assert.deepEqual(await snapshotOf(store), ["t/a:7", "t/b:2"], "a record the batch names is replaced"); + await store.pushAll([]); + assert.deepEqual(await snapshotOf(store), ["t/a:7", "t/b:2"], "an empty flush from a cache that holds nothing wipes nothing"); + const before2 = [...(await readFile(path))]; + await assert.rejects(() => store.pushAll([entryOf("a", "u", 1)]), SchemaConflictError); + assert.deepEqual([...(await readFile(path))], before2, "a batch that retypes a held key is refused and the file is unchanged"); } finally { await rm(directory, { recursive: true, force: true }); } diff --git a/packages/cultmesh-ts/test/cultmesh.test.ts b/packages/cultmesh-ts/test/cultmesh.test.ts index 10de21b9..b22d5d65 100644 --- a/packages/cultmesh-ts/test/cultmesh.test.ts +++ b/packages/cultmesh-ts/test/cultmesh.test.ts @@ -109,6 +109,19 @@ test("CultMesh TS opens a durable local node and registers document bindings", a assert.ok(reopened.documents.getBySchemaId("cultmesh.note.v0")); }); +test("CultMesh TS flush from a node that never pulled leaves the records the store holds", async () => { + const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-flush-")), "node.ccmp"); + const first = await CultMesh.startNode(filePath, { documents: [noteDocument] }); + await first.put(noteDocument, "note:held", { noteId: "note:held", body: "already in the store" }); + await first.flush(); + + const unpulled = await CultMesh.startNode(filePath, { documents: [noteDocument], pullOnStart: false }); + await unpulled.flush(); + + const reopened = await CultMesh.startNode(filePath, { documents: [noteDocument] }); + assert.equal(reopened.getRequired(noteDocument, "note:held").body, "already in the store"); +}); + test("CultMesh TS document handles hide local cache plumbing behind typed reactive reads", async () => { const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-doc-")), "node.ccmp"); const node = await CultMesh.startNode(filePath, {