From aa8d8ddd6b2b28e7853f9bf39a9fffbee14e76fe Mon Sep 17 00:00:00 2001 From: soul Date: Tue, 10 Nov 2026 23:47:18 +0000 Subject: [PATCH] cultcache-ts: opening a legacy store is a pure read; no heal-on-open Delete the pullAll rewrite and SingleFileStoreRead.repairedLegacyPayload (no consumer). The file becomes v1 on the next push/delete/flush. Tests use a synthetic pre-v1 vector; Python and Rust pins added. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-py/tests/test_cultcache.py | 19 +++++ packages/cultcache-rs/src/lib.rs | 21 ++++++ packages/cultcache-ts/CHANGELOG.md | 11 ++- packages/cultcache-ts/README.md | 2 +- .../single-file-messagepack-backing-store.ts | 37 ++-------- packages/cultcache-ts/test/cult-cache.test.ts | 72 +++++++++++-------- packages/cultcache-ts/test/store-file.test.ts | 56 +++++++++++---- 7 files changed, 139 insertions(+), 79 deletions(-) diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index f5bad365..1dd41a0e 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -1121,6 +1121,25 @@ class CultCacheTests(unittest.TestCase): catalog_entry=catalog_entry, ) + def test_pulling_a_synthetic_pre_v1_envelope_array_leaves_the_file_bytes_identical(self) -> None: + try: + import msgpack # type: ignore + except ImportError: + self.skipTest("msgpack optional dependency is not installed") + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "legacy.cc" + legacy = msgpack.packb( + [{"key": "note:hello", "type": "note", "payload": {"title": "Hello"}, "storedAt": "2026-10-20T00:00:00Z"}], + use_bin_type=True, + ) + path.write_bytes(legacy) + + envelopes = SingleFileMessagePackBackingStore(path).pull_all() + + self.assertEqual([e.key for e in envelopes], ["note:hello"]) + self.assertEqual(path.read_bytes(), legacy) + self.assertEqual(sorted(p.name for p in Path(tmp).iterdir()), ["legacy.cc"]) + def test_legacy_key_global_loads_without_writing_and_first_write_leaves_one_global(self) -> None: with tempfile.TemporaryDirectory() as tmp: settings = define_document_type("settings", global_document=True) diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 4ceb43f2..ce7756e3 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -6136,6 +6136,27 @@ mod tests { Ok(()) } + #[test] + fn pulling_a_synthetic_pre_v1_envelope_array_leaves_the_file_bytes_identical() -> Result<()> { + // SYNTHETIC pre-v1 vector: no real file of this shape is known; the Rust envelope payload is binary. + let temp = tempfile::tempdir()?; + let path = temp.path().join("legacy.cc"); + let legacy = rmp_serde::to_vec_named(&vec![CultCacheEnvelope { + key: "note:hello".to_string(), + r#type: "note".to_string(), + payload: vec![0x90], + stored_at: "2026-10-20T00:00:00Z".to_string(), + schema_id: None, + }])?; + fs::write(&path, &legacy)?; + + let envelopes = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + + assert_eq!(envelopes.iter().map(|e| e.key.as_str()).collect::>(), ["note:hello"]); + assert_eq!(fs::read(&path)?, legacy); + Ok(()) + } + #[test] fn a_newer_store_is_named_once_its_header_string_is_complete() -> Result<()> { let vector = fs::read( diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 819df6c4..eefea3b5 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Changed + +- Opening a legacy (pre-v1) store is a pure read: payloads that were not bytes are normalized in memory and the + file is left as it was, so a codec that refuses the content cannot lose the old bytes. The next push, delete or + flush rewrites the file as v1, merging what the open read. +- `SingleFileStoreRead` no longer carries `repairedLegacyPayload`; no consumer used it. + ## [0.15.0] 0.14.1 and 0.14.2 were never tagged or published; this entry carries them with the Unreleased work, and `cultcache-ts-v0.14.0` is the previous release. @@ -41,8 +50,6 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - The Windows replace is a POSIX-semantics rename first (it lands while C# and Rust readers hold the store open), falling back to a write-through `MoveFileExW`, with 25 attempts 20 ms apart on errors 5, 32 and 33. Writers wait up to 30 s for the lock. -- Opening a legacy store that needs healing rewrites it under the lock, from a read taken - under the lock. ### Dependencies diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index b3a85345..dc53c42d 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -357,7 +357,7 @@ That is the TypeScript version of the original CultCache trick: - the top-level MessagePack value is `[formatVersion, schemaCatalog, records]` - each record is `[key, schemaId, storedAt, payload]` - each catalog entry is `[schemaId, schemaName, schemaVersion, contentHash, canonicalSchemaJson, compatibleSchemaIds, members]` -- it still reads the older TS/Rust-style envelope array and rewrites repaired legacy payloads as v1 snapshots +- it still reads the older TS/Rust-style envelope array (object payloads are normalized in memory; opening never rewrites the file) and replaces it with a v1 snapshot on the next push, delete or flush - it queues local writes in order, then takes the shared `.lock` (`flock` on Unix, `LockFileEx` on Windows, via koffi, loaded on the first write) for each one, so C#, Rust and TypeScript writers exclude each other; the read a write merges from is taken under that lock - it fsyncs the staged file and, on Unix, the directory after the replace; on Windows it replaces with a POSIX-semantics rename and falls back to a write-through `MoveFileExW`, retrying briefly on sharing violations - it persists raw MessagePack payload bytes inside schema-addressed records diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 2501c632..6428ef48 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -47,21 +47,11 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } async pullAll(): Promise { - // The disk decides the header: a file that is gone or legacy is not marked. - this.#format = STORE_FORMAT_VERSION; + // A pure read: the disk decides the header, and a legacy file is normalized in memory only. The file is rewritten + // as v1 by the next push, delete or flush, which merges what this read returns; opening never replaces bytes the + // consumer's codec has not yet accepted. const disk = this.#readDisk(); this.#format = disk.format; - if (disk.repairedLegacyPayload) { - // The heal decides from its own read, taken under the store lock. - await this.#enqueue(async () => { - const again = this.#readDisk(); - this.#format = again.format; - if (again.repairedLegacyPayload) { - await this.#writeAll([], again.envelopes); - } - }); - } - return disk.envelopes; } @@ -158,7 +148,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { export function readSingleFileStore(path: string): SingleFileStoreRead { const data = readStoreBytes(path); if (data === undefined) { - return { absent: true, format: STORE_FORMAT_VERSION, envelopes: [], repairedLegacyPayload: false }; + return { absent: true, format: STORE_FORMAT_VERSION, envelopes: [] }; } try { @@ -395,8 +385,6 @@ export type SingleFileStoreRead = { absent: boolean; format: StoreFormat; envelopes: CultCacheEnvelope[]; - // A legacy file whose payloads were not bytes: reading it repairs them, and the store rewrites the file. - repairedLegacyPayload: boolean; }; function decodeStoreFile(data: Uint8Array): Omit { @@ -420,7 +408,7 @@ function decodeStoreFile(data: Uint8Array): Omit storedAt: record.storedAt, }; }); - return { format: snapshot.format, envelopes, repairedLegacyPayload: false }; + return { format: snapshot.format, envelopes }; } const legacy = decodeLegacyEnvelopeArray(decoded); @@ -428,19 +416,8 @@ function decodeStoreFile(data: Uint8Array): Omit throw new Error("CultCache file is not a recognized CultCache MessagePack store."); } - let repairedLegacyPayload = false; - const normalized = legacy.map((entry) => { - const payload = normalizePayload(entry.payload); - if (payload !== entry.payload) { - repairedLegacyPayload = true; - } - - return { - ...entry, - payload, - }; - }); - return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[], repairedLegacyPayload }; + const normalized = legacy.map((entry) => ({ ...entry, payload: normalizePayload(entry.payload) })); + return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[] }; } function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined { diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 02324273..de0cad79 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -504,46 +504,56 @@ test("SingleFileMessagePackBackingStore refuses a record whose schema the catalo await assert.rejects(() => cache.pullAllBackingStores(), StoreUnreadableError); }); -test("SingleFileMessagePackBackingStore heals legacy envelopes whose payload was persisted as an object", async () => { - const noteDocument = defineDocumentType({ - type: "note", - schema: z.object({ - title: z.string(), - body: z.string(), - }), - }); +const legacyNoteType = defineDocumentType({ + type: "note", + schema: z.object({ title: z.string(), body: z.string().optional() }), +}); +// SYNTHETIC pre-v1 file (no real one is known): an envelope array with object payloads. +async function legacyNoteStore(second: unknown): Promise<{ storePath: string; bytes: Uint8Array }> { const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "legacy.msgpack"); - const envelopePayload = { - title: "Hello", - body: "world", - }; - - await writeFile( - storePath, - encode([ - { - key: "note:hello", - type: "note", - payload: envelopePayload, - storedAt: new Date().toISOString(), - }, - ]), - ); + const bytes = encode([ + { key: "note:hello", type: "note", payload: { title: "Hello", body: "world" }, storedAt: "2026-10-20T00:00:00Z" }, + { key: "note:other", type: "note", payload: second, storedAt: "2026-10-20T00:00:00Z" }, + ]); + await writeFile(storePath, bytes); + return { storePath, bytes }; +} +test("a codec that refuses a synthetic pre-v1 store's content leaves the file byte-identical", async () => { + const { storePath, bytes } = await legacyNoteStore({ title: 42 }); const cache = CultCache.builder() - .withDocumentType(noteDocument) + .withDocumentType(legacyNoteType) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + + await assert.rejects(() => cache.pullAllBackingStores()); + assert.deepEqual([...(await readFile(storePath))], [...bytes]); + assert.deepEqual((await readdir(dirname(storePath))).sort(), ["legacy.msgpack"]); +}); + +test("a synthetic pre-v1 store is read untouched and becomes v1 only when the consumer writes, keeping its other entries", async () => { + const { storePath, bytes } = await legacyNoteStore({ title: "Other" }); + const cache = CultCache.builder() + .withDocumentType(legacyNoteType) .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) .build(); await cache.pullAllBackingStores(); - assert.deepEqual(cache.getRequired(noteDocument, "note:hello"), envelopePayload); + assert.deepEqual(cache.getRequired(legacyNoteType, "note:hello"), { title: "Hello", body: "world" }); + assert.deepEqual([...(await readFile(storePath))], [...bytes], "reading does not rewrite the file"); - const rewritten = decode(await readFile(storePath)) as unknown[]; - assert.equal(rewritten[0], "cultcache.store.v1"); - const records = rewritten[2] as unknown[][]; - assert.equal(records.length, 1); - assert.ok(records[0]?.[3] instanceof Uint8Array); + await cache.put(legacyNoteType, "note:new", { title: "New" }); + assert.equal((decode(await readFile(storePath)) as unknown[])[0], "cultcache.store.v1"); + + const reopened = CultCache.builder() + .withDocumentType(legacyNoteType) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await reopened.pullAllBackingStores(); + assert.deepEqual(reopened.getRequired(legacyNoteType, "note:hello"), { title: "Hello", body: "world" }); + assert.deepEqual(reopened.getRequired(legacyNoteType, "note:other"), { title: "Other" }); + assert.deepEqual(reopened.getRequired(legacyNoteType, "note:new"), { title: "New" }); }); test("CultCache rejects a second generic backing store", async () => { diff --git a/packages/cultcache-ts/test/store-file.test.ts b/packages/cultcache-ts/test/store-file.test.ts index e98309f1..39ac553f 100644 --- a/packages/cultcache-ts/test/store-file.test.ts +++ b/packages/cultcache-ts/test/store-file.test.ts @@ -471,44 +471,70 @@ test("the store lock is released when the operation throws", async () => { } }); +// SYNTHETIC pre-v1 vector: no real file of this shape is known to exist. A top-level array of {key,type,payload,storedAt} +// whose payload is not binary (an object, a Node Buffer JSON, an int array) is what the removed heal-on-open rewrote. const legacyObjectPayload = (): Uint8Array => encode([{ key: "note:hello", type: "note", payload: { title: "Hello" }, storedAt: "2026-10-20T00:00:00Z" }]); -test("an open that heals a legacy store waits for the store lock and heals from its own read", async () => { +test("opening a synthetic pre-v1 store with object payloads returns them normalized and leaves the file untouched, even while another holder has the lock", async () => { const directory = await scratch(); try { const path = join(directory, "legacy.cc"); const legacy = legacyObjectPayload(); await writeFile(path, legacy); const store = new SingleFileMessagePackBackingStore(path); - let healed: Promise | undefined; - let settled = false; await withStoreWriteLock(path, async () => { - healed = store.pullAll().then(() => { - settled = true; - }); - await sleepFor(300); - assert.equal(settled, false, "the heal must wait for the lock"); - assert.deepEqual([...(await readFile(path))], [...legacy], "the file keeps its legacy bytes while the lock is held elsewhere"); + const [lockHeld] = await Promise.all([readdir(directory)]); + const envelopes = await within(store.pullAll(), 5000, "an open does not wait for the write lock"); + assert.deepEqual(envelopes.map((entry) => entry.key), ["note:hello"]); + assert.deepEqual(decode(documentPayloadBytes(envelopes[0].payload)), { title: "Hello" }); + assert.deepEqual([...(await readFile(path))], [...legacy]); + assert.deepEqual((await readdir(directory)).sort(), lockHeld.sort(), "an open adds no lock or temp file"); }); - await within(healed!, 5000, "the heal"); - assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1"); + assert.deepEqual([...(await readFile(path))], [...legacy]); + assert.deepEqual((await readdir(directory)).filter((name) => name !== "legacy.cc" && name !== "legacy.cc.lock"), []); } finally { await rm(directory, { recursive: true, force: true }); } }); -test("push and delete over a legacy store that needs healing finish instead of waiting on their own lock", async () => { +test("opening a synthetic pre-v1 store with Buffer-JSON and int-array payloads normalizes them in memory and leaves the file untouched", async () => { + const directory = await scratch(); + try { + const path = join(directory, "legacy.cc"); + const inner = encode({ title: "Hello" }); + const legacy = encode([ + { key: "a", type: "note", payload: { type: "Buffer", data: [...inner] }, storedAt: "2026-10-20T00:00:00Z" }, + { key: "b", type: "note", payload: [...inner], storedAt: "2026-10-20T00:00:00Z" }, + ]); + await writeFile(path, legacy); + const envelopes = await new SingleFileMessagePackBackingStore(path).pullAll(); + assert.deepEqual(envelopes.map((entry) => entry.key), ["a", "b"]); + for (const envelope of envelopes) { + assert.deepEqual(decode(documentPayloadBytes(envelope.payload)), { title: "Hello" }); + } + assert.deepEqual([...(await readFile(path))], [...legacy]); + assert.deepEqual(await readdir(directory), ["legacy.cc"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("push and delete over a synthetic pre-v1 store write v1 and keep the other legacy entries, normalized payloads included", async () => { const directory = await scratch(); try { const path = join(directory, "legacy.cc"); await writeFile(path, legacyObjectPayload()); const store = new SingleFileMessagePackBackingStore(path); await within(store.push(entryOf("a")), 5000, "push"); + assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1"); + assert.deepEqual((await store.pullAll()).map((entry) => entry.key).sort(), ["a", "note:hello"]); await writeFile(path, legacyObjectPayload()); await within(store.delete(entryOf("a")), 5000, "delete"); - const keys = (await store.pullAll()).map((entry) => entry.key); - assert.deepEqual(keys, ["note:hello"]); + assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1"); + const kept = await store.pullAll(); + assert.deepEqual(kept.map((entry) => entry.key), ["note:hello"]); + assert.deepEqual(decode(documentPayloadBytes(kept[0].payload)), { title: "Hello" }); } finally { await rm(directory, { recursive: true, force: true }); } @@ -697,7 +723,7 @@ test("a store file of an unrecognized shape is refused, and a store naming a mis } }); -test("opening a store that needs no healing writes nothing: a missing store leaves no lock file, a legacy store with byte payloads keeps its bytes", async () => { +test("opening writes nothing: a missing store leaves no lock file, a legacy store with byte payloads keeps its bytes", async () => { const directory = await scratch(); try { const absent = join(directory, "nested", "absent.cc"); -- 2.47.3