From 4028205ff155d3574a40b58378fcb9d841039faf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 01:59:28 +0200 Subject: [PATCH] Store readability: one reader decides open, flush, commit and CultMesh writes A rewrite replaces a file only when the runtime's own reader opens it. C# drops ReadStoreHeader (a second, header-only verdict that let a flush replace files the reader refuses) and asks DeserializeSnapshot from open, flush and commit; the reader now requires exactly one array, so C# refuses bytes after the store on read as TS and Python do. A zero-byte file is an empty store for all three paths. CultMesh.WriteSingleFileDocument reads the file it replaces through the Mesh reader before writing. Rust reads and rewrites through one classification (trailing bytes refused on read; a non-store first slot must be a legacy envelope array the reader accepts). TypeScript checks a legacy array with pullAll's own decoder before replacing it. Python already conformed. Shared byte vectors and per-runtime verdicts live in tests/vectors/document-variants-c2a/readability; every runtime's test reads them. Holds gains second-dictionary-value and second-list-item cases. The persistence format contract now marks v3 by content and states the readability rule. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-py/tests/test_cultcache.py | 56 +++---- packages/cultcache-rs/src/lib.rs | 139 +++++++----------- .../single-file-messagepack-backing-store.ts | 16 +- packages/cultcache-ts/test/cult-cache.test.ts | 60 ++++---- .../CultDocumentMessagePackSerialization.cs | 61 +++----- .../Contracts/cultcache-persistence-format.md | 32 ++-- src/GameCult.Caching/CultCache.cs | 28 ++-- .../CultMeshSingleFileDocuments.cs | 14 +- .../ElementIdEngineTests.cs | 4 + .../GameCult.Caching.Tests/ElementIdTests.cs | 60 -------- .../StoreReadabilityTests.cs | 115 +++++++++++++++ .../CultMeshSingleFileReadabilityTests.cs | 75 ++++++++++ .../readability/.gitattributes | 1 + .../readability/empty-array.bin | 1 + .../readability/generate.py | 33 +++++ .../readability/header-v9.bin | 1 + .../readability/ints.bin | 1 + .../readability/legacy-envelopes.bin | 1 + .../readability/manifest.txt | 16 ++ .../document-variants-c2a/readability/map.bin | 1 + .../readability/nested-array.bin | 1 + .../readability/nil-first.bin | 1 + .../readability/scalar.bin | 1 + .../readability/string-first.bin | 1 + .../readability/trailing-bytes.bin | 1 + .../readability/trailing-store.bin | 1 + .../readability/truncated.bin | 1 + .../readability/zero-byte.bin | 0 28 files changed, 452 insertions(+), 270 deletions(-) create mode 100644 tests/GameCult.Caching.Tests/StoreReadabilityTests.cs create mode 100644 tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs create mode 100644 tests/vectors/document-variants-c2a/readability/.gitattributes create mode 100644 tests/vectors/document-variants-c2a/readability/empty-array.bin create mode 100644 tests/vectors/document-variants-c2a/readability/generate.py create mode 100644 tests/vectors/document-variants-c2a/readability/header-v9.bin create mode 100644 tests/vectors/document-variants-c2a/readability/ints.bin create mode 100644 tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin create mode 100644 tests/vectors/document-variants-c2a/readability/manifest.txt create mode 100644 tests/vectors/document-variants-c2a/readability/map.bin create mode 100644 tests/vectors/document-variants-c2a/readability/nested-array.bin create mode 100644 tests/vectors/document-variants-c2a/readability/nil-first.bin create mode 100644 tests/vectors/document-variants-c2a/readability/scalar.bin create mode 100644 tests/vectors/document-variants-c2a/readability/string-first.bin create mode 100644 tests/vectors/document-variants-c2a/readability/trailing-bytes.bin create mode 100644 tests/vectors/document-variants-c2a/readability/trailing-store.bin create mode 100644 tests/vectors/document-variants-c2a/readability/truncated.bin create mode 100644 tests/vectors/document-variants-c2a/readability/zero-byte.bin diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index e1d28354..5241ea14 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -903,34 +903,38 @@ class CultCacheTests(unittest.TestCase): SingleFileMessagePackBackingStore(store_path).push_all(envelopes) self.assertEqual(store_path.read_bytes(), before, f"{vector.name} was rewritten") - def test_single_file_push_all_rewrites_only_a_file_that_is_exactly_one_store(self) -> None: - import msgpack - + # A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by + # pull_all, push_all, push and delete alike. The bytes and every runtime's verdict: + # tests/vectors/document-variants-c2a/readability. + def test_single_file_replaces_a_file_exactly_when_it_reads(self) -> None: + root = self._C2A_VECTORS / "readability" + rows = [ + line.split() + for line in (root / "manifest.txt").read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ] + self.assertEqual(len(rows), 14) envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all() - whole = (self._C2A_VECTORS / "v3-base.msgpack").read_bytes() - refused = { - "truncated": whole[:-1], - "trailing": whole + bytes([1, 2, 3]), - "scalar": bytes([0x01]), - "map": bytes([0x80]), - "string first": msgpack.packb(["hello", 1]), - } - for name, content in refused.items(): - with tempfile.TemporaryDirectory() as tmp: - store_path = Path(tmp) / "store.msgpack" - store_path.write_bytes(content) - with self.assertRaises(ValueError, msg=name): - SingleFileMessagePackBackingStore(store_path).push_all(envelopes) - self.assertEqual(store_path.read_bytes(), content, f"{name} was rewritten") + for vector, _, _, _, python in rows: + content = (root / vector).read_bytes() + for operation in ("pull_all", "push_all", "push", "delete"): + with tempfile.TemporaryDirectory() as tmp: + store_path = Path(tmp) / "store.msgpack" + store_path.write_bytes(content) + store = SingleFileMessagePackBackingStore(store_path) + run = { + "pull_all": store.pull_all, + "push_all": lambda: store.push_all(envelopes), + "push": lambda: store.push(envelopes[0]), + "delete": lambda: store.delete(envelopes[0].type, envelopes[0].key), + }[operation] + if python == "reads": + run() + else: + with self.assertRaises(Exception, msg=f"{vector} {operation}"): + run() + self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read") - # An empty file and a legacy envelope array (empty included) carry no header and are rewritten unmarked. - legacy = msgpack.packb([{"key": "old", "type": "old", "payload": b"", "storedAt": "then"}], use_bin_type=True) - for name, content in {"empty": b"", "empty array": bytes([0x90]), "legacy": legacy}.items(): - with tempfile.TemporaryDirectory() as tmp: - store_path = Path(tmp) / "store.msgpack" - store_path.write_bytes(content) - SingleFileMessagePackBackingStore(store_path).push_all(envelopes) - self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], "cultcache.store.v1", name) if __name__ == "__main__": diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 9bff75f1..fed9ecdf 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -319,11 +319,10 @@ const STORE_FORMAT_V1: &str = "cultcache.store.v1"; /// found under it; it does not decide when a document carries ids. const STORE_FORMAT_ELEMENT_IDS: &str = "cultcache.store.v3"; -/// The store header string, `None` for the legacy envelope array (its first slot is not a string), and an error for -/// bytes that are not one complete MessagePack array: a truncated store is refused as the snapshot reader refuses it, -/// never taken for a legacy file. One pass reads the first slot and walks the rest to prove the file whole. A rewrite -/// asks for the file to be `whole`: exactly that one array, so bytes after it are refused too. -fn store_header(bytes: &[u8], whole: bool) -> Result> { +/// The store header string, `None` when the first slot is not a string (a legacy envelope array candidate), and an error +/// for bytes that are not exactly one complete MessagePack array: a truncated store or bytes after the array are refused, +/// never taken for a legacy file. One pass reads the first slot and walks the rest to prove the file whole. +fn store_header(bytes: &[u8]) -> Result> { #[derive(serde::Deserialize)] #[serde(untagged)] enum FirstSlot { @@ -358,7 +357,7 @@ fn store_header(bytes: &[u8], whole: bool) -> Result> { let header =
::deserialize(&mut rmp_serde::Deserializer::new(&mut cursor)) .map_err(|error| anyhow!("the store is not one complete MessagePack array: {error}"))?; let after = bytes.len() - cursor.position() as usize; - ensure!(!whole || after == 0, "the store has {after} bytes after its MessagePack array"); + ensure!(after == 0, "the store has {after} bytes after its MessagePack array"); Ok(header.0) } @@ -741,10 +740,13 @@ impl SingleFileMessagePackBackingStore { if bytes.is_empty() { return Ok(Vec::new()); } - match store_header(&bytes, false).ok().flatten() { - Some(header) if header.starts_with("cultcache.store.") => decode_store_snapshot(&bytes), - _ => rmp_serde::from_slice(&bytes).map_err(anyhow::Error::from), - } + (|| match store_header(&bytes)? { + Some(header) => { + readable_store_format(&header)?; + decode_store_snapshot(&bytes) + } + None => decode_legacy_envelopes(&bytes), + })() .map_err(|error| anyhow!("failed to decode MessagePack {}: {error:#}", self.path.display())) } @@ -772,11 +774,15 @@ impl SingleFileMessagePackBackingStore { // would drop them. let format = match fs::read(&self.path) { Ok(bytes) if bytes.is_empty() => STORE_FORMAT_V1, - Ok(bytes) => match store_header(&bytes, true) + Ok(bytes) => match store_header(&bytes) .map_err(|error| anyhow!("{}: {error:#}; the store is not rewritten", self.path.display()))? { Some(header) => readable_store_format(&header)?, - None => STORE_FORMAT_V1, + None => { + decode_legacy_envelopes(&bytes) + .map_err(|error| anyhow!("{}: {error:#}; the store is not rewritten", self.path.display()))?; + STORE_FORMAT_V1 + } }, Err(error) if error.kind() == std::io::ErrorKind::NotFound => STORE_FORMAT_V1, Err(error) => return Err(error).with_context(|| format!("failed to read {}", self.path.display())), @@ -2627,10 +2633,13 @@ fn readable_store_format(header: &str) -> Result<&'static str> { } } +/// The legacy envelope array: an array of envelope maps. Its acceptance is the one verdict on an array whose first slot +/// is not a store header, for reading it and for replacing it. +fn decode_legacy_envelopes(bytes: &[u8]) -> Result> { + rmp_serde::from_slice(bytes).map_err(anyhow::Error::from) +} + fn decode_store_snapshot(bytes: &[u8]) -> Result> { - if let Ok(Some(header)) = store_header(bytes, false) { - readable_store_format(&header)?; - } let snapshot: PersistedStoreSnapshot = rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?; @@ -4755,7 +4764,7 @@ mod tests { let keys: Vec<_> = envelopes.iter().map(|e| e.key.as_str()).collect(); assert_eq!(keys, vec!["alpha", "beta"]); store.push(&envelopes[0])?; - Ok(store_header(&std::fs::read(&path)?, true)?.expect("a store header")) + Ok(store_header(&std::fs::read(&path)?)?.expect("a store header")) } #[test] @@ -4817,75 +4826,39 @@ mod tests { } - // A store cut short is not a legacy file: the rewrite refuses it whatever its header says, and leaves it be. + // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by open, + // push_all and push alike. The bytes and every runtime's verdict are tests/vectors/document-variants-c2a/readability. #[test] - fn push_all_refuses_a_store_truncated_by_one_byte_and_leaves_it_untouched() -> Result<()> { - for vector in [ - "document-variants-c0/variant-v2.msgpack", - "document-variants-c1/variant-store.msgpack", - "document-variants-c2a/v3-base.msgpack", - ] { - let temp = tempfile::tempdir()?; - let path = temp.path().join("store.msgpack"); - let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors").join(vector); - let mut bytes = std::fs::read(source)?; - bytes.pop(); - std::fs::write(&path, &bytes)?; - let mut store = SingleFileMessagePackBackingStore::new(&path); - let message = format!( - "{:#}", - store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()).unwrap_err() - ); - assert!(message.contains("not rewritten"), "{vector}: {message}"); - assert_eq!(std::fs::read(&path)?, bytes, "{vector} was rewritten"); + fn a_file_is_replaced_exactly_when_it_reads() -> Result<()> { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/readability"); + let manifest = std::fs::read_to_string(root.join("manifest.txt"))?; + let mut rows = 0; + for line in manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) { + let cells: Vec<&str> = line.split_whitespace().collect(); + let (vector, reads) = (cells[0], cells[2] == "reads"); + let bytes = std::fs::read(root.join(vector))?; + rows += 1; + for operation in ["open", "push_all", "push"] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + std::fs::write(&path, &bytes)?; + let mut store = SingleFileMessagePackBackingStore::new(&path); + let outcome = match operation { + "open" => store.pull_all().map(|_| ()), + "push_all" => store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()), + _ => store.push(&snapshot_envelope("x", b"one")), + }; + let what = format!("{vector} {operation}"); + assert_eq!(outcome.is_ok(), reads, "{what}: {outcome:?}"); + if !reads { + assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a file it cannot read"); + } else if operation != "open" { + assert!(store_header(&std::fs::read(&path)?)?.is_some(), "{what} left a file that is not a store"); + assert!(store.pull_all()?.iter().any(|entry| entry.key == "x"), "{what}"); + } + } } - Ok(()) - } - // A rewrite replaces only a file that is exactly one store: bytes after the array, a value that is not an array, or a - // first slot that is a string but not a header are refused and left be. An empty file and a legacy envelope array - // (first slot not a string, empty included) carry no header and are written v1. - #[test] - fn push_all_rewrites_only_a_file_that_is_exactly_one_store() -> Result<()> { - let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/v3-base.msgpack"); - let mut trailing = std::fs::read(source)?; - trailing.extend_from_slice(&[1, 2, 3]); - let refused: Vec<(&str, Vec)> = vec![ - ("trailing", trailing), - ("scalar", vec![0x01]), - ("map", vec![0x80]), - ("string first", rmp_serde::to_vec(&("hello", 1))?), - ]; - for (name, bytes) in refused { - let temp = tempfile::tempdir()?; - let path = temp.path().join("store.msgpack"); - std::fs::write(&path, &bytes)?; - let mut store = SingleFileMessagePackBackingStore::new(&path); - let result = store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()); - assert!(result.is_err(), "{name} was accepted"); - assert_eq!(std::fs::read(&path)?, bytes, "{name} was rewritten"); - } - - for (name, bytes) in [("empty array", vec![0x90u8]), ("legacy", vec![0x91u8, 0x80])] { - let temp = tempfile::tempdir()?; - let path = temp.path().join("store.msgpack"); - std::fs::write(&path, &bytes)?; - let mut store = SingleFileMessagePackBackingStore::new(&path); - store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default())?; - assert_eq!(store_header(&std::fs::read(&path)?, true)?.as_deref(), Some(STORE_FORMAT_V1), "{name}"); - } - Ok(()) - } - - // A zero-byte file is an empty store with no header, not a truncated one: a rewrite writes it v1. - #[test] - fn push_all_rewrites_a_zero_byte_file_as_v1() -> Result<()> { - let temp = tempfile::tempdir()?; - let path = temp.path().join("store.msgpack"); - std::fs::write(&path, [])?; - let mut store = SingleFileMessagePackBackingStore::new(&path); - store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default())?; - assert_eq!(store_header(&std::fs::read(&path)?, true)?.as_deref(), Some(STORE_FORMAT_V1)); - assert_eq!(store.pull_all()?.len(), 1); + assert_eq!(rows, 14); Ok(()) } diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 4aafea26..bd351511 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -143,8 +143,8 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } // A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids - // stays marked, and one that is not (or is gone, empty or legacy) is written unmarked. A file that is not exactly one - // store this runtime reads is refused and left as it is. + // stays marked, and one that is not (or is gone, empty or legacy) is written unmarked. A file `pullAll` would refuse + // (not exactly one store, not a legacy envelope array) is refused and left as it is. await this.#readDiskFormat(); await this.#writeAll(entries); }); @@ -153,7 +153,17 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { async #readDiskFormat(): Promise { try { const data = await readFile(this.filePath); - this.#format = (data.length === 0 ? undefined : storeHeader(decode(data))) ?? STORE_FORMAT_VERSION; + let header: StoreFormat | undefined; + if (data.length > 0) { + const decoded = decode(data); + header = storeHeader(decoded); + if (header === undefined) { + // Not a store header: only a legacy envelope array is replaced, and only one `pullAll` would read. + decodeLegacyEnvelopeArray(decoded); + } + } + + this.#format = header ?? STORE_FORMAT_VERSION; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { throw error; diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 027f1f43..7235a6c4 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -1582,35 +1582,39 @@ test("SingleFileMessagePackBackingStore pushAll refuses a store whose header it } }); -test("SingleFileMessagePackBackingStore pushAll rewrites only a file that is exactly one store and leaves anything else untouched", async () => { - const dir = await mkdtemp(join(tmpdir(), "cultcache-pushall-whole-")); +// A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by pullAll, +// pushAll, push and delete alike. The bytes and every runtime's verdict: tests/vectors/document-variants-c2a/readability. +test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", async () => { + const root = join(c2aVectors, "readability"); + const rows = (await readFile(join(root, "manifest.txt"), "utf8")) + .split(/\r?\n/u) + .filter((line) => line.length > 0 && !line.startsWith("#")) + .map((line) => line.split(/\s+/u)); + assert.equal(rows.length, 14); + const dir = await mkdtemp(join(tmpdir(), "cultcache-readability-")); const seed = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "seed.msgpack"); - const whole = await readFile(seed); const envelopes = await new SingleFileMessagePackBackingStore(seed).pullAll(); - const refused: Array<[string, Uint8Array]> = [ - ["truncated", whole.subarray(0, whole.length - 1)], - ["trailing", Buffer.concat([whole, Buffer.from([1, 2, 3])])], - ["scalar", Uint8Array.of(0x01)], - ["map", Uint8Array.of(0x80)], - ["string-first", encode(["hello", 1])], - ]; - for (const [name, bytes] of refused) { - const file = join(dir, `${name}.msgpack`); - await writeFile(file, bytes); - await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pushAll(envelopes), Error, name); - assert.ok(Buffer.from(bytes).equals(await readFile(file)), `${name} was rewritten`); - } - - // An empty file and a legacy envelope array (first slot not a string, empty included) carry no header and are rewritten. - const accepted: Array<[string, Uint8Array]> = [ - ["empty", new Uint8Array()], - ["empty-array", Uint8Array.of(0x90)], - ["legacy", encode([{ key: "old", type: "old", payload: new Uint8Array(), storedAt: "then" }])], - ]; - for (const [name, bytes] of accepted) { - const file = join(dir, `${name}.msgpack`); - await writeFile(file, bytes); - await new SingleFileMessagePackBackingStore(file).pushAll(envelopes); - assert.equal((decode(await readFile(file)) as unknown[])[0], "cultcache.store.v1", name); + for (const [vector, , , typescript] of rows) { + const reads = typescript === "reads"; + const bytes = await readFile(join(root, vector)); + for (const operation of ["pullAll", "pushAll", "push", "delete"] as const) { + const file = join(dir, `${vector.replace(/[^a-z0-9.-]/giu, "_")}.${operation}.msgpack`); + await writeFile(file, bytes); + const store = new SingleFileMessagePackBackingStore(file); + const run = () => + operation === "pullAll" + ? store.pullAll() + : operation === "pushAll" + ? store.pushAll(envelopes) + : operation === "push" + ? store.push(envelopes[0]!) + : store.delete(envelopes[0]!); + if (reads) { + await run(); + } else { + await assert.rejects(run, Error, `${vector} ${operation}`); + assert.ok(bytes.equals(await readFile(file)), `${vector} ${operation} rewrote a file it cannot read`); + } + } } }); diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs index 631c9d6a..d1a4df52 100644 --- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs +++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs @@ -177,8 +177,14 @@ public static class CultDocumentMessagePackSerialization return buffer.WrittenSpan.ToArray(); } + /// + /// Reads a store file: exactly one complete MessagePack array. A truncated file, bytes after the array, a value that is not + /// an array, or a first slot that is not a header string are refused, so a reader never takes part of a file for the whole. + /// The rewriting paths use this same reader as their verdict on whether a file may be replaced. + /// public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload) { + RequireOneArray(payload); var reader = new MessagePackReader(payload); var fieldCount = reader.ReadArrayHeader(); var snapshot = new CultPersistedStoreSnapshot(); @@ -217,6 +223,24 @@ public static class CultDocumentMessagePackSerialization return snapshot; } + private static void RequireOneArray(byte[] bytes) + { + var whole = new MessagePackReader(bytes); + try + { + if (whole.NextMessagePackType != MessagePackType.Array) + throw new NotSupportedException($"The store is a MessagePack {whole.NextMessagePackType}, not an array."); + whole.Skip(); + } + catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException) + { + throw new NotSupportedException("The store is not one complete MessagePack array.", ex); + } + + if (!whole.End) + throw new NotSupportedException("The store has bytes after its MessagePack array."); + } + /// Refuses a single-file snapshot this runtime cannot read, naming the format or record it found. public static void RequireSingleFileFormat(CultPersistedStoreSnapshot snapshot) { @@ -241,41 +265,6 @@ public static class CultDocumentMessagePackSerialization } } - /// - /// The format header of a store file about to be replaced by a writer that does not read the store's records: null for - /// an empty file or a legacy envelope array (an array whose first slot is not a string), the header string otherwise. - /// A file is a store only when it is exactly one complete MessagePack array: a truncated file, bytes after the array, a - /// value that is not an array, or a first slot that is a string but not a header this runtime reads are all refused, - /// never taken for a legacy file, so a writer cannot overwrite what it cannot see. - /// - public static string? ReadStoreHeader(byte[] bytes) - { - if (bytes.Length == 0) - return null; - var whole = new MessagePackReader(bytes); - try - { - if (whole.NextMessagePackType != MessagePackType.Array) - throw new NotSupportedException($"The store is a MessagePack {whole.NextMessagePackType}, not an array; it is not rewritten."); - whole.Skip(); - } - catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException) - { - throw new NotSupportedException("The store is not one complete MessagePack array; it is not rewritten.", ex); - } - - if (!whole.End) - throw new NotSupportedException("The store has bytes after its MessagePack array; it is not rewritten."); - - var reader = new MessagePackReader(bytes); - if (reader.ReadArrayHeader() == 0 || reader.NextMessagePackType != MessagePackType.String) - return null; - // A first slot that is a string is a header, and one this runtime does not read is refused, whatever the string says. - var header = reader.ReadString() ?? string.Empty; - RequireSingleFileFormat(new CultPersistedStoreSnapshot { FormatVersion = header }); - return header; - } - private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record) { writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount); @@ -568,8 +557,6 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore return snapshot; } - protected override string? ReadStoreHeader(byte[] data) => CultDocumentMessagePackSerialization.ReadStoreHeader(data); - protected override byte[] SerializePayload(object document) { return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry); diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index f998b8ea..d887e44e 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -219,12 +219,26 @@ A reader refuses any `cultcache.store.*` header it does not know and any record slots than its header allows, naming the version or the record's key and schema id. It never skips a slot it does not understand. -The headers are `v1` (no variant, no element ids), `v2` (holds a variant) and `v3` (can -hold element ids, variants or not). A single-file store declares `v3` when any record it -holds is of a type that reaches an object list, and a file already declared `v3` stays -`v3` when a writer merges onto it. A directory store's manifest says -`cultcache.store.v5.directory-content-addressed-pages` under the same rule, and -`v4` otherwise. +The headers are `v1` (no variant, no element ids), `v2` (holds a variant) and `v3` (holds +an element id, variants or not). A single-file store declares `v3` when any record it +holds carries an element id: a non-empty `[CultElementId]` value in a record's document, +or in a variant's override values. A store that cannot read a variant's override values +(no codec attached) counts the variant as holding one. A whole-store writer decides by +what the store holds now, so the file drops back to `v1` (or `v2` with a variant) when +the last id is taken away; a writer that touches only part of the file cannot see the +rest, so a file already declared `v3` stays `v3` when it merges onto it. A directory +store's manifest says `cultcache.store.v5.directory-content-addressed-pages` under the +same rule, and `v4` otherwise. + +A single-file store is exactly one MessagePack array. Every runtime refuses a file that is +truncated, has bytes after the array (a second store appended included), is not an array, or +whose first slot is not a header it reads; it refuses these on open and on rewrite alike. A +zero-byte file is an empty store. A flush or a commit replaces a file only when the +runtime's own reader opens it, so one verdict covers open, flush and commit, and a file the +runtime cannot read is left as it is. Rust, TypeScript and Python also open the legacy +envelope array (an array of `key`/`type`/`payload`/`storedAt` maps) and replace it; C# has +no legacy reader and refuses it. The shared byte vectors and each runtime's verdict are +`tests/vectors/document-variants-c2a/readability/manifest.txt`. ## Variant Records @@ -245,8 +259,8 @@ fifth slot: `elementId` is `""` unless the step enters an object-list element. `value` is the member's own MessagePack encoding, inline. A top-level Set has one step and `id` `""`; a reader that resolves only that shape refuses the others, naming the variant. -- A store holding at least one variant declares `cultcache.store.v2` (`v3` when it can - hold element ids); a store holding none declares `cultcache.store.v1` (or `v3`) and is +- A store holding at least one variant declares `cultcache.store.v2` (`v3` when it holds an + element id); a store holding none declares `cultcache.store.v1` (or `v3`) and is byte-identical to what a runtime without variants writes. A `v1` header over a variant record is refused. - A variant's resolved document is derived in memory when the cache admits the record @@ -276,7 +290,7 @@ elements retargets nothing. written before ids existed loads, and its ids are minted from SHA-256 of the record key and the element's path, so every reader mints the same ids; they reach disk with the record's next write, or `CultCache.MintElementIds()`. -- A store that can hold ids declares the `v3` header (above), so a reader older than ids +- A store that holds an id declares the `v3` header (above), so a reader older than ids refuses it instead of skipping the id slot and rewriting the elements without ids. TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they rewrite a store; they do not decide when a document carries ids. diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index f66cde48..e12ee0b9 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -3416,15 +3416,13 @@ namespace GameCult.Caching protected FileInfo FileInfo { get; } protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot); - protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data); /// - /// The format header of a non-empty store file this store is about to replace: null for a legacy file that carries none, - /// the header otherwise. It throws for bytes that are not exactly one store this runtime reads, so a rewrite never - /// overwrites a file it cannot see. The default reads the whole snapshot; a codec that can read the header alone - /// overrides it. + /// This store's one reader of a non-empty store file, and so its one verdict on whether the file may be replaced: open, + /// flush and commit all ask it. It must throw for anything that is not exactly one store this runtime reads (a + /// truncated file, bytes after the store, a format it does not know), so a rewrite never overwrites a file it cannot see. /// - protected virtual string? ReadStoreHeader(byte[] data) => DeserializeSnapshot(data).FormatVersion; + protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data); protected abstract byte[] SerializePayload(object document); protected abstract object DeserializePayload(Type documentType, byte[] payload); @@ -3502,8 +3500,7 @@ namespace GameCult.Caching { using (AcquireLock(wait: true)) { - if (ReadDisk() is { Length: > 0 } onDisk) - ReadStoreHeader(onDisk); + ReadSnapshot(); WriteSnapshot( Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()), @@ -3529,7 +3526,7 @@ namespace GameCult.Caching if (fileLock == null) return CultCommitOutcome.Contended; - var disk = ReadSnapshot(rewriting: true) ?? new CultPersistedStoreSnapshot(); + var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot(); if (!request.ConditionsHold(disk.Records, Entries.Values)) return CultCommitOutcome.Mismatch; @@ -3598,15 +3595,12 @@ namespace GameCult.Caching Judging!(arriving, departing); } - // A commit rewrites the file it read, so it reads it as a rewrite does: the header proves the bytes are one store first. - private CultPersistedStoreSnapshot? ReadSnapshot(bool rewriting = false) + // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is + // gone or zero bytes is an empty store: nothing to read, nothing to lose. + private CultPersistedStoreSnapshot? ReadSnapshot() { var bytes = ReadDisk(); - if (bytes == null) - return null; - if (rewriting && bytes.Length > 0) - ReadStoreHeader(bytes); - return DeserializeSnapshot(bytes); + return bytes == null ? null : DeserializeSnapshot(bytes); } private byte[]? ReadDisk() @@ -3616,7 +3610,7 @@ namespace GameCult.Caching return null; try { - return ReadAllBytesShared(FileInfo.FullName); + return ReadAllBytesShared(FileInfo.FullName) is { Length: > 0 } bytes ? bytes : null; } catch (FileNotFoundException) { diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs index ae033e54..2a2ff2f6 100644 --- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs +++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs @@ -241,11 +241,15 @@ namespace GameCult.Mesh throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry)); payload ??= Array.Empty(); + // The file being replaced is read first, by the reader every read of it uses: one this runtime cannot read refuses the + // write instead of being overwritten. + var existing = File.Exists(path) ? ReadSingleFileSnapshot(path) : null; + // A typed write sees the document it replaces the file with, so its ids decide. A raw payload is opaque: the writer // cannot see whether it holds ids, so a file already marked stays marked. var snapshot = new CultPersistedStoreSnapshot { - FormatVersion = CacheBackingStore.HeaderFor(holdsIds, contentKnown ? null : ExistingHeader(path), wholeStore: contentKnown, directoryStore: false), + FormatVersion = CacheBackingStore.HeaderFor(holdsIds, existing?.FormatVersion, wholeStore: contentKnown, directoryStore: false), SchemaCatalog = new[] { catalogEntry }, Records = new[] { @@ -266,15 +270,11 @@ namespace GameCult.Mesh WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot)); } - private static string? ExistingHeader(string path) - { - // A file that is gone has no header; one that cannot be read refuses the write rather than being overwritten. - return File.Exists(path) ? CultDocumentMessagePackSerialization.ReadStoreHeader(File.ReadAllBytes(path)) : null; - } - private static CultPersistedStoreSnapshot ReadSingleFileSnapshot(string path) { var bytes = File.ReadAllBytes(path); + if (bytes.Length == 0) + return new CultPersistedStoreSnapshot(); CultPersistedStoreSnapshot snapshot; try { diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 2016b975..dae82cee 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -83,6 +83,10 @@ namespace GameCult.Caching.Tests Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists"); Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element"); Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element"); + Assert.That(CultElementIds.Holds(new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value"); + Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list"); + Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements"); + Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list"); } [Test] diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index fb941453..37d938e3 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -785,66 +785,6 @@ namespace GameCult.Caching.Tests Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "read without one, it cannot be seen, so it is marked"); } - // A flush replaces the whole file, but only a file that is exactly one store this runtime reads (Rust, TypeScript and Python - // refuse the same): anything else throws and is left as it was. An empty file and a legacy envelope array (an array whose - // first slot is not a string, empty included) carry no header and are replaced. - [TestCase("truncated", false)] - [TestCase("v9", false)] - [TestCase("trailing", false)] - [TestCase("scalar", false)] - [TestCase("map", false)] - [TestCase("string-first", false)] - [TestCase("empty", true)] - [TestCase("legacy", true)] - [TestCase("empty-array", true)] - public void AFlushRefusesAFileThatIsNotExactlyOneStoreAndLeavesItUntouched(string file, bool replaced) - { - var path = PathOf($"flush-over-{file}.cc"); - using var cache = Open(path); - cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); - var bytes = NotAStore(file, File.ReadAllBytes(path)); - File.WriteAllBytes(path, bytes); - - if (replaced) - { - cache.BackingStores[0].PushAll(); - Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the file is this store's view now"); - return; - } - - Assert.Throws(() => cache.BackingStores[0].PushAll()); - Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "the file was rewritten"); - } - - // An unconditional commit rewrites the file it read: a store with bytes after its array is refused, not rewritten. - [Test] - public void AnUnconditionalCommitRefusesAStoreWithBytesAfterItsArray() - { - var path = PathOf("commit-over-trailing.cc"); - using var cache = Open(path); - cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d"))); - var bytes = NotAStore("trailing", File.ReadAllBytes(path)); - File.WriteAllBytes(path, bytes); - - Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("e"), new CultRecordKey("e")))); - Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "the file was rewritten"); - } - - // What a store file is not, built from a whole store file. - private static byte[] NotAStore(string file, byte[] whole) => file switch - { - "truncated" => whole[..^1], - "v9" => MessagePackSerializer.Serialize(new object[] { "cultcache.store.v9", Array.Empty(), Array.Empty() }), - "trailing" => whole.Concat(new byte[] { 1, 2, 3 }).ToArray(), - "scalar" => new byte[] { 0x01 }, - "map" => new byte[] { 0x80 }, - "string-first" => MessagePackSerializer.Serialize(new object[] { "hello", 1 }), - "empty" => Array.Empty(), - "empty-array" => new byte[] { 0x90 }, - "legacy" => MessagePackSerializer.Serialize(new object[] { new Dictionary { ["key"] = "old", ["type"] = "old", ["storedAt"] = "then" } }), - _ => throw new ArgumentOutOfRangeException(nameof(file)) - }; - [Test] public void ADirectoryStoreIsMarkedByTheIdsItWritesAndAnEmptyDeckDoesNotMarkIt() { diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs new file mode 100644 index 00000000..9f2cc074 --- /dev/null +++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs @@ -0,0 +1,115 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using GameCult.Caching.MessagePack; +using NUnit.Framework; +using static GameCult.Caching.Tests.ElementIdTests; + +namespace GameCult.Caching.Tests +{ + // A file is replaced by a flush or a commit exactly when this runtime's own reader opens it: one verdict per file, asked by + // open, flush and commit alike. The bytes and the verdict of every runtime are shared: tests/vectors/document-variants-c2a/readability. + public class StoreReadabilityTests + { + private const int CSharp = 1; + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) }); + private string _directory = ""; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "cultlib-readability-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, true); + } + + private static string VectorRoot() + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a", "readability"); + if (Directory.Exists(candidate)) + return candidate; + } + + throw new DirectoryNotFoundException($"Shared readability vectors not found above {TestContext.CurrentContext.TestDirectory}."); + } + + public static IEnumerable Vectors() => + File.ReadAllLines(Path.Combine(VectorRoot(), "manifest.txt")) + .Where(line => line.Length > 0 && line[0] != '#') + .Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}")); + + private static string Header(string path) => CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion; + + private string Seed(string name) + { + var path = Path.Combine(_directory, name); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "d" }, new CultRecordKey("d"))); + return path; + } + + [TestCaseSource(nameof(Vectors))] + public void OpenReadsExactlyTheFilesTheVectorsSayItReads(string vector, bool reads) + { + var path = Path.Combine(_directory, "open.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), vector))); + + using var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + if (reads) + Assert.DoesNotThrow(() => cache.PullAllBackingStoresAsync().GetAwaiter().GetResult()); + else + Assert.That(() => cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(), Throws.Exception); + } + + [TestCaseSource(nameof(Vectors))] + public void AFlushReplacesAFileExactlyWhenItReads(string vector, bool reads) + { + var path = Seed("flush.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector)); + File.WriteAllBytes(path, bytes); + var store = cache.BackingStores[0]; + + if (reads) + { + store.PushAll(); + Assert.That(Header(path), Does.StartWith("cultcache.store.v"), "the file is a store this runtime wrote"); + return; + } + + Assert.That(() => store.PushAll(), Throws.Exception); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten"); + } + + [TestCaseSource(nameof(Vectors))] + public void AnUnconditionalCommitReplacesAFileExactlyWhenItReads(string vector, bool reads) + { + var path = Seed("commit.cc"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector)); + File.WriteAllBytes(path, bytes); + + if (reads) + { + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))); + Assert.That(Header(path), Does.StartWith("cultcache.store.v"), "the file is a store this runtime wrote"); + return; + } + + Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.Exception); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten"); + } + } +} diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs new file mode 100644 index 00000000..d9cc7ef7 --- /dev/null +++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FluentAssertions; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +#nullable enable + +namespace GameCult.Mesh.Tests; + +// A single-file write replaces a file exactly when the runtime's own reader opens it, as a cache's flush does. The bytes and +// the verdict are shared with every runtime: tests/vectors/document-variants-c2a/readability. +public sealed class CultMeshSingleFileReadabilityTests +{ + private const int CSharp = 1; + private string _root = null!; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "cultmesh-single-file-readability", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_root)) Directory.Delete(_root, true); + } + + private static string VectorRoot() + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a", "readability"); + if (Directory.Exists(candidate)) return candidate; + } + + throw new DirectoryNotFoundException($"Shared readability vectors not found above {TestContext.CurrentContext.TestDirectory}."); + } + + public static IEnumerable Vectors() => + File.ReadAllLines(Path.Combine(VectorRoot(), "manifest.txt")) + .Where(line => line.Length > 0 && line[0] != '#') + .Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}")); + + private static CultMeshBodyPublicationDocument Publication() => new() + { + BodyId = "aetheria:entities", + ProducerId = "aetheria", + SchemaId = "eve.entity_soa.v1" + }; + + [TestCaseSource(nameof(Vectors))] + public void ATypedWriteReplacesAFileExactlyWhenItReads(string vector, bool reads) + { + var path = Path.Combine(_root, "doc.cc"); + var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector)); + File.WriteAllBytes(path, bytes); + + if (reads) + { + CultMesh.WriteSingleFileDocument(path, new CultRecordKey("publication"), Publication()); + CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Should().ContainSingle(); + return; + } + + Assert.That(() => CultMesh.WriteSingleFileDocument(path, new CultRecordKey("publication"), Publication()), Throws.Exception); + File.ReadAllBytes(path).Should().Equal(bytes, "a file this runtime cannot read was rewritten"); + } +} diff --git a/tests/vectors/document-variants-c2a/readability/.gitattributes b/tests/vectors/document-variants-c2a/readability/.gitattributes new file mode 100644 index 00000000..291377b6 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/.gitattributes @@ -0,0 +1 @@ +*.bin binary diff --git a/tests/vectors/document-variants-c2a/readability/empty-array.bin b/tests/vectors/document-variants-c2a/readability/empty-array.bin new file mode 100644 index 00000000..9e99dd5e --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/empty-array.bin @@ -0,0 +1 @@ +� \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/generate.py b/tests/vectors/document-variants-c2a/readability/generate.py new file mode 100644 index 00000000..bb63d0a9 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/generate.py @@ -0,0 +1,33 @@ +"""Generates the shared store-readability vectors. Every runtime's tests read the committed bytes and the verdicts in +manifest.txt; nothing regenerates them at test time. Run: python generate.py (needs msgpack). + +A file may be replaced by a flush or a commit exactly when the runtime's own reader opens it, so one verdict per runtime +covers open, flush and commit. Each vector is the bytes named; the trailing and truncated ones are cut from v3-base. +""" +import pathlib + +import msgpack + +here = pathlib.Path(__file__).resolve().parent +whole = (here.parent / "v3-base.msgpack").read_bytes() +pack = lambda value: msgpack.packb(value, use_bin_type=True) + +vectors = { + "zero-byte": b"", + "empty-array": pack([]), + # A legacy envelope array: an array of maps. Rust, TypeScript and Python read it; C# has no legacy reader. + "legacy-envelopes": pack([{"key": "old", "type": "old.type", "payload": pack([]), "storedAt": "2026-01-01T00:00:00Z"}]), + "ints": pack([1, 2, 3]), + "nil-first": pack([None, "x"]), + "nested-array": pack([["cultcache.store.v9", [], []]]), + "string-first": pack(["hello", 1]), + "header-v9": pack(["cultcache.store.v9", [], []]), + "scalar": b"\x01", + "map": b"\x80", + "truncated": whole[:-1], + "trailing-bytes": whole + b"\x01\x02\x03", + "trailing-store": whole + whole, +} +for name, data in vectors.items(): + (here / f"{name}.bin").write_bytes(data) +print(len(vectors), "vectors") diff --git a/tests/vectors/document-variants-c2a/readability/header-v9.bin b/tests/vectors/document-variants-c2a/readability/header-v9.bin new file mode 100644 index 00000000..42f9eba0 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/header-v9.bin @@ -0,0 +1 @@ +“²cultcache.store.v9�� \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/ints.bin b/tests/vectors/document-variants-c2a/readability/ints.bin new file mode 100644 index 00000000..dfedbc05 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/ints.bin @@ -0,0 +1 @@ +“ \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin b/tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin new file mode 100644 index 00000000..a55b231f --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin @@ -0,0 +1 @@ +‘„£key£old¤type¨old.type§payloadÄ�¨storedAt´2026-01-01T00:00:00Z \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/manifest.txt b/tests/vectors/document-variants-c2a/readability/manifest.txt new file mode 100644 index 00000000..55e028f9 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/manifest.txt @@ -0,0 +1,16 @@ +# Whether each runtime's own reader opens the file. A flush and a commit replace a file exactly when it reads. +# Columns: file, csharp, rust, typescript, python. `../v3-base.msgpack` is the one valid store. +../v3-base.msgpack reads reads reads reads +zero-byte.bin reads reads reads reads +empty-array.bin reads reads reads reads +legacy-envelopes.bin refuses reads reads reads +ints.bin refuses refuses refuses refuses +nil-first.bin refuses refuses refuses refuses +nested-array.bin refuses refuses refuses refuses +string-first.bin refuses refuses refuses refuses +header-v9.bin refuses refuses refuses refuses +scalar.bin refuses refuses refuses refuses +map.bin refuses refuses refuses refuses +truncated.bin refuses refuses refuses refuses +trailing-bytes.bin refuses refuses refuses refuses +trailing-store.bin refuses refuses refuses refuses diff --git a/tests/vectors/document-variants-c2a/readability/map.bin b/tests/vectors/document-variants-c2a/readability/map.bin new file mode 100644 index 00000000..5416677b --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/map.bin @@ -0,0 +1 @@ +€ \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/nested-array.bin b/tests/vectors/document-variants-c2a/readability/nested-array.bin new file mode 100644 index 00000000..34b09870 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/nested-array.bin @@ -0,0 +1 @@ +‘“²cultcache.store.v9�� \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/nil-first.bin b/tests/vectors/document-variants-c2a/readability/nil-first.bin new file mode 100644 index 00000000..5ce298f2 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/nil-first.bin @@ -0,0 +1 @@ +’À¡x \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/scalar.bin b/tests/vectors/document-variants-c2a/readability/scalar.bin new file mode 100644 index 00000000..6b2aaa76 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/scalar.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/string-first.bin b/tests/vectors/document-variants-c2a/readability/string-first.bin new file mode 100644 index 00000000..862a69f7 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/string-first.bin @@ -0,0 +1 @@ +’¥hello \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/trailing-bytes.bin b/tests/vectors/document-variants-c2a/readability/trailing-bytes.bin new file mode 100644 index 00000000..d191fcfe --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/trailing-bytes.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/trailing-store.bin b/tests/vectors/document-variants-c2a/readability/trailing-store.bin new file mode 100644 index 00000000..2d3ca7ec --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/trailing-store.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/truncated.bin b/tests/vectors/document-variants-c2a/readability/truncated.bin new file mode 100644 index 00000000..96796824 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/truncated.bin @@ -0,0 +1 @@ +“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta \ No newline at end of file diff --git a/tests/vectors/document-variants-c2a/readability/zero-byte.bin b/tests/vectors/document-variants-c2a/readability/zero-byte.bin new file mode 100644 index 00000000..e69de29b