diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py
index e1d28354..5241ea14 100644
--- a/packages/cultcache-py/tests/test_cultcache.py
+++ b/packages/cultcache-py/tests/test_cultcache.py
@@ -903,34 +903,38 @@ class CultCacheTests(unittest.TestCase):
SingleFileMessagePackBackingStore(store_path).push_all(envelopes)
self.assertEqual(store_path.read_bytes(), before, f"{vector.name} was rewritten")
- def test_single_file_push_all_rewrites_only_a_file_that_is_exactly_one_store(self) -> None:
- import msgpack
-
+ # A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by
+ # pull_all, push_all, push and delete alike. The bytes and every runtime's verdict:
+ # tests/vectors/document-variants-c2a/readability.
+ def test_single_file_replaces_a_file_exactly_when_it_reads(self) -> None:
+ root = self._C2A_VECTORS / "readability"
+ rows = [
+ line.split()
+ for line in (root / "manifest.txt").read_text(encoding="utf-8").splitlines()
+ if line and not line.startswith("#")
+ ]
+ self.assertEqual(len(rows), 14)
envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all()
- whole = (self._C2A_VECTORS / "v3-base.msgpack").read_bytes()
- refused = {
- "truncated": whole[:-1],
- "trailing": whole + bytes([1, 2, 3]),
- "scalar": bytes([0x01]),
- "map": bytes([0x80]),
- "string first": msgpack.packb(["hello", 1]),
- }
- for name, content in refused.items():
- with tempfile.TemporaryDirectory() as tmp:
- store_path = Path(tmp) / "store.msgpack"
- store_path.write_bytes(content)
- with self.assertRaises(ValueError, msg=name):
- SingleFileMessagePackBackingStore(store_path).push_all(envelopes)
- self.assertEqual(store_path.read_bytes(), content, f"{name} was rewritten")
+ for vector, _, _, _, python in rows:
+ content = (root / vector).read_bytes()
+ for operation in ("pull_all", "push_all", "push", "delete"):
+ with tempfile.TemporaryDirectory() as tmp:
+ store_path = Path(tmp) / "store.msgpack"
+ store_path.write_bytes(content)
+ store = SingleFileMessagePackBackingStore(store_path)
+ run = {
+ "pull_all": store.pull_all,
+ "push_all": lambda: store.push_all(envelopes),
+ "push": lambda: store.push(envelopes[0]),
+ "delete": lambda: store.delete(envelopes[0].type, envelopes[0].key),
+ }[operation]
+ if python == "reads":
+ run()
+ else:
+ with self.assertRaises(Exception, msg=f"{vector} {operation}"):
+ run()
+ self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read")
- # An empty file and a legacy envelope array (empty included) carry no header and are rewritten unmarked.
- legacy = msgpack.packb([{"key": "old", "type": "old", "payload": b"", "storedAt": "then"}], use_bin_type=True)
- for name, content in {"empty": b"", "empty array": bytes([0x90]), "legacy": legacy}.items():
- with tempfile.TemporaryDirectory() as tmp:
- store_path = Path(tmp) / "store.msgpack"
- store_path.write_bytes(content)
- SingleFileMessagePackBackingStore(store_path).push_all(envelopes)
- self.assertEqual(msgpack.unpackb(store_path.read_bytes(), raw=False)[0], "cultcache.store.v1", name)
if __name__ == "__main__":
diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs
index 9bff75f1..fed9ecdf 100644
--- a/packages/cultcache-rs/src/lib.rs
+++ b/packages/cultcache-rs/src/lib.rs
@@ -319,11 +319,10 @@ const STORE_FORMAT_V1: &str = "cultcache.store.v1";
/// found under it; it does not decide when a document carries ids.
const STORE_FORMAT_ELEMENT_IDS: &str = "cultcache.store.v3";
-/// The store header string, `None` for the legacy envelope array (its first slot is not a string), and an error for
-/// bytes that are not one complete MessagePack array: a truncated store is refused as the snapshot reader refuses it,
-/// never taken for a legacy file. One pass reads the first slot and walks the rest to prove the file whole. A rewrite
-/// asks for the file to be `whole`: exactly that one array, so bytes after it are refused too.
-fn store_header(bytes: &[u8], whole: bool) -> Result> {
+/// The store header string, `None` when the first slot is not a string (a legacy envelope array candidate), and an error
+/// for bytes that are not exactly one complete MessagePack array: a truncated store or bytes after the array are refused,
+/// never taken for a legacy file. One pass reads the first slot and walks the rest to prove the file whole.
+fn store_header(bytes: &[u8]) -> Result > {
#[derive(serde::Deserialize)]
#[serde(untagged)]
enum FirstSlot {
@@ -358,7 +357,7 @@ fn store_header(bytes: &[u8], whole: bool) -> Result > {
let header = ::deserialize(&mut rmp_serde::Deserializer::new(&mut cursor))
.map_err(|error| anyhow!("the store is not one complete MessagePack array: {error}"))?;
let after = bytes.len() - cursor.position() as usize;
- ensure!(!whole || after == 0, "the store has {after} bytes after its MessagePack array");
+ ensure!(after == 0, "the store has {after} bytes after its MessagePack array");
Ok(header.0)
}
@@ -741,10 +740,13 @@ impl SingleFileMessagePackBackingStore {
if bytes.is_empty() {
return Ok(Vec::new());
}
- match store_header(&bytes, false).ok().flatten() {
- Some(header) if header.starts_with("cultcache.store.") => decode_store_snapshot(&bytes),
- _ => rmp_serde::from_slice(&bytes).map_err(anyhow::Error::from),
- }
+ (|| match store_header(&bytes)? {
+ Some(header) => {
+ readable_store_format(&header)?;
+ decode_store_snapshot(&bytes)
+ }
+ None => decode_legacy_envelopes(&bytes),
+ })()
.map_err(|error| anyhow!("failed to decode MessagePack {}: {error:#}", self.path.display()))
}
@@ -772,11 +774,15 @@ impl SingleFileMessagePackBackingStore {
// would drop them.
let format = match fs::read(&self.path) {
Ok(bytes) if bytes.is_empty() => STORE_FORMAT_V1,
- Ok(bytes) => match store_header(&bytes, true)
+ Ok(bytes) => match store_header(&bytes)
.map_err(|error| anyhow!("{}: {error:#}; the store is not rewritten", self.path.display()))?
{
Some(header) => readable_store_format(&header)?,
- None => STORE_FORMAT_V1,
+ None => {
+ decode_legacy_envelopes(&bytes)
+ .map_err(|error| anyhow!("{}: {error:#}; the store is not rewritten", self.path.display()))?;
+ STORE_FORMAT_V1
+ }
},
Err(error) if error.kind() == std::io::ErrorKind::NotFound => STORE_FORMAT_V1,
Err(error) => return Err(error).with_context(|| format!("failed to read {}", self.path.display())),
@@ -2627,10 +2633,13 @@ fn readable_store_format(header: &str) -> Result<&'static str> {
}
}
+/// The legacy envelope array: an array of envelope maps. Its acceptance is the one verdict on an array whose first slot
+/// is not a store header, for reading it and for replacing it.
+fn decode_legacy_envelopes(bytes: &[u8]) -> Result> {
+ rmp_serde::from_slice(bytes).map_err(anyhow::Error::from)
+}
+
fn decode_store_snapshot(bytes: &[u8]) -> Result> {
- if let Ok(Some(header)) = store_header(bytes, false) {
- readable_store_format(&header)?;
- }
let snapshot: PersistedStoreSnapshot =
rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?;
@@ -4755,7 +4764,7 @@ mod tests {
let keys: Vec<_> = envelopes.iter().map(|e| e.key.as_str()).collect();
assert_eq!(keys, vec!["alpha", "beta"]);
store.push(&envelopes[0])?;
- Ok(store_header(&std::fs::read(&path)?, true)?.expect("a store header"))
+ Ok(store_header(&std::fs::read(&path)?)?.expect("a store header"))
}
#[test]
@@ -4817,75 +4826,39 @@ mod tests {
}
- // A store cut short is not a legacy file: the rewrite refuses it whatever its header says, and leaves it be.
+ // A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by open,
+ // push_all and push alike. The bytes and every runtime's verdict are tests/vectors/document-variants-c2a/readability.
#[test]
- fn push_all_refuses_a_store_truncated_by_one_byte_and_leaves_it_untouched() -> Result<()> {
- for vector in [
- "document-variants-c0/variant-v2.msgpack",
- "document-variants-c1/variant-store.msgpack",
- "document-variants-c2a/v3-base.msgpack",
- ] {
- let temp = tempfile::tempdir()?;
- let path = temp.path().join("store.msgpack");
- let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors").join(vector);
- let mut bytes = std::fs::read(source)?;
- bytes.pop();
- std::fs::write(&path, &bytes)?;
- let mut store = SingleFileMessagePackBackingStore::new(&path);
- let message = format!(
- "{:#}",
- store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()).unwrap_err()
- );
- assert!(message.contains("not rewritten"), "{vector}: {message}");
- assert_eq!(std::fs::read(&path)?, bytes, "{vector} was rewritten");
+ fn a_file_is_replaced_exactly_when_it_reads() -> Result<()> {
+ let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/readability");
+ let manifest = std::fs::read_to_string(root.join("manifest.txt"))?;
+ let mut rows = 0;
+ for line in manifest.lines().filter(|line| !line.is_empty() && !line.starts_with('#')) {
+ let cells: Vec<&str> = line.split_whitespace().collect();
+ let (vector, reads) = (cells[0], cells[2] == "reads");
+ let bytes = std::fs::read(root.join(vector))?;
+ rows += 1;
+ for operation in ["open", "push_all", "push"] {
+ let temp = tempfile::tempdir()?;
+ let path = temp.path().join("store.msgpack");
+ std::fs::write(&path, &bytes)?;
+ let mut store = SingleFileMessagePackBackingStore::new(&path);
+ let outcome = match operation {
+ "open" => store.pull_all().map(|_| ()),
+ "push_all" => store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default()),
+ _ => store.push(&snapshot_envelope("x", b"one")),
+ };
+ let what = format!("{vector} {operation}");
+ assert_eq!(outcome.is_ok(), reads, "{what}: {outcome:?}");
+ if !reads {
+ assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a file it cannot read");
+ } else if operation != "open" {
+ assert!(store_header(&std::fs::read(&path)?)?.is_some(), "{what} left a file that is not a store");
+ assert!(store.pull_all()?.iter().any(|entry| entry.key == "x"), "{what}");
+ }
+ }
}
- Ok(())
- }
- // A rewrite replaces only a file that is exactly one store: bytes after the array, a value that is not an array, or a
- // first slot that is a string but not a header are refused and left be. An empty file and a legacy envelope array
- // (first slot not a string, empty included) carry no header and are written v1.
- #[test]
- fn push_all_rewrites_only_a_file_that_is_exactly_one_store() -> Result<()> {
- let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/document-variants-c2a/v3-base.msgpack");
- let mut trailing = std::fs::read(source)?;
- trailing.extend_from_slice(&[1, 2, 3]);
- let refused: Vec<(&str, Vec)> = vec![
- ("trailing", trailing),
- ("scalar", vec![0x01]),
- ("map", vec![0x80]),
- ("string first", rmp_serde::to_vec(&("hello", 1))?),
- ];
- for (name, bytes) in refused {
- let temp = tempfile::tempdir()?;
- let path = temp.path().join("store.msgpack");
- std::fs::write(&path, &bytes)?;
- let mut store = SingleFileMessagePackBackingStore::new(&path);
- let result = store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default());
- assert!(result.is_err(), "{name} was accepted");
- assert_eq!(std::fs::read(&path)?, bytes, "{name} was rewritten");
- }
-
- for (name, bytes) in [("empty array", vec![0x90u8]), ("legacy", vec![0x91u8, 0x80])] {
- let temp = tempfile::tempdir()?;
- let path = temp.path().join("store.msgpack");
- std::fs::write(&path, &bytes)?;
- let mut store = SingleFileMessagePackBackingStore::new(&path);
- store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default())?;
- assert_eq!(store_header(&std::fs::read(&path)?, true)?.as_deref(), Some(STORE_FORMAT_V1), "{name}");
- }
- Ok(())
- }
-
- // A zero-byte file is an empty store with no header, not a truncated one: a rewrite writes it v1.
- #[test]
- fn push_all_rewrites_a_zero_byte_file_as_v1() -> Result<()> {
- let temp = tempfile::tempdir()?;
- let path = temp.path().join("store.msgpack");
- std::fs::write(&path, [])?;
- let mut store = SingleFileMessagePackBackingStore::new(&path);
- store.push_all(&[snapshot_envelope("x", b"one")], PushAllOptions::default())?;
- assert_eq!(store_header(&std::fs::read(&path)?, true)?.as_deref(), Some(STORE_FORMAT_V1));
- assert_eq!(store.pull_all()?.len(), 1);
+ assert_eq!(rows, 14);
Ok(())
}
diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts
index 4aafea26..bd351511 100644
--- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts
+++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts
@@ -143,8 +143,8 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
}
// A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids
- // stays marked, and one that is not (or is gone, empty or legacy) is written unmarked. A file that is not exactly one
- // store this runtime reads is refused and left as it is.
+ // stays marked, and one that is not (or is gone, empty or legacy) is written unmarked. A file `pullAll` would refuse
+ // (not exactly one store, not a legacy envelope array) is refused and left as it is.
await this.#readDiskFormat();
await this.#writeAll(entries);
});
@@ -153,7 +153,17 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
async #readDiskFormat(): Promise {
try {
const data = await readFile(this.filePath);
- this.#format = (data.length === 0 ? undefined : storeHeader(decode(data))) ?? STORE_FORMAT_VERSION;
+ let header: StoreFormat | undefined;
+ if (data.length > 0) {
+ const decoded = decode(data);
+ header = storeHeader(decoded);
+ if (header === undefined) {
+ // Not a store header: only a legacy envelope array is replaced, and only one `pullAll` would read.
+ decodeLegacyEnvelopeArray(decoded);
+ }
+ }
+
+ this.#format = header ?? STORE_FORMAT_VERSION;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts
index 027f1f43..7235a6c4 100644
--- a/packages/cultcache-ts/test/cult-cache.test.ts
+++ b/packages/cultcache-ts/test/cult-cache.test.ts
@@ -1582,35 +1582,39 @@ test("SingleFileMessagePackBackingStore pushAll refuses a store whose header it
}
});
-test("SingleFileMessagePackBackingStore pushAll rewrites only a file that is exactly one store and leaves anything else untouched", async () => {
- const dir = await mkdtemp(join(tmpdir(), "cultcache-pushall-whole-"));
+// A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by pullAll,
+// pushAll, push and delete alike. The bytes and every runtime's verdict: tests/vectors/document-variants-c2a/readability.
+test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", async () => {
+ const root = join(c2aVectors, "readability");
+ const rows = (await readFile(join(root, "manifest.txt"), "utf8"))
+ .split(/\r?\n/u)
+ .filter((line) => line.length > 0 && !line.startsWith("#"))
+ .map((line) => line.split(/\s+/u));
+ assert.equal(rows.length, 14);
+ const dir = await mkdtemp(join(tmpdir(), "cultcache-readability-"));
const seed = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "seed.msgpack");
- const whole = await readFile(seed);
const envelopes = await new SingleFileMessagePackBackingStore(seed).pullAll();
- const refused: Array<[string, Uint8Array]> = [
- ["truncated", whole.subarray(0, whole.length - 1)],
- ["trailing", Buffer.concat([whole, Buffer.from([1, 2, 3])])],
- ["scalar", Uint8Array.of(0x01)],
- ["map", Uint8Array.of(0x80)],
- ["string-first", encode(["hello", 1])],
- ];
- for (const [name, bytes] of refused) {
- const file = join(dir, `${name}.msgpack`);
- await writeFile(file, bytes);
- await assert.rejects(() => new SingleFileMessagePackBackingStore(file).pushAll(envelopes), Error, name);
- assert.ok(Buffer.from(bytes).equals(await readFile(file)), `${name} was rewritten`);
- }
-
- // An empty file and a legacy envelope array (first slot not a string, empty included) carry no header and are rewritten.
- const accepted: Array<[string, Uint8Array]> = [
- ["empty", new Uint8Array()],
- ["empty-array", Uint8Array.of(0x90)],
- ["legacy", encode([{ key: "old", type: "old", payload: new Uint8Array(), storedAt: "then" }])],
- ];
- for (const [name, bytes] of accepted) {
- const file = join(dir, `${name}.msgpack`);
- await writeFile(file, bytes);
- await new SingleFileMessagePackBackingStore(file).pushAll(envelopes);
- assert.equal((decode(await readFile(file)) as unknown[])[0], "cultcache.store.v1", name);
+ for (const [vector, , , typescript] of rows) {
+ const reads = typescript === "reads";
+ const bytes = await readFile(join(root, vector));
+ for (const operation of ["pullAll", "pushAll", "push", "delete"] as const) {
+ const file = join(dir, `${vector.replace(/[^a-z0-9.-]/giu, "_")}.${operation}.msgpack`);
+ await writeFile(file, bytes);
+ const store = new SingleFileMessagePackBackingStore(file);
+ const run = () =>
+ operation === "pullAll"
+ ? store.pullAll()
+ : operation === "pushAll"
+ ? store.pushAll(envelopes)
+ : operation === "push"
+ ? store.push(envelopes[0]!)
+ : store.delete(envelopes[0]!);
+ if (reads) {
+ await run();
+ } else {
+ await assert.rejects(run, Error, `${vector} ${operation}`);
+ assert.ok(bytes.equals(await readFile(file)), `${vector} ${operation} rewrote a file it cannot read`);
+ }
+ }
}
});
diff --git a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
index 631c9d6a..d1a4df52 100644
--- a/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
+++ b/src/GameCult.Caching.MessagePack/CultDocumentMessagePackSerialization.cs
@@ -177,8 +177,14 @@ public static class CultDocumentMessagePackSerialization
return buffer.WrittenSpan.ToArray();
}
+ ///
+ /// Reads a store file: exactly one complete MessagePack array. A truncated file, bytes after the array, a value that is not
+ /// an array, or a first slot that is not a header string are refused, so a reader never takes part of a file for the whole.
+ /// The rewriting paths use this same reader as their verdict on whether a file may be replaced.
+ ///
public static CultPersistedStoreSnapshot DeserializeSnapshot(byte[] payload)
{
+ RequireOneArray(payload);
var reader = new MessagePackReader(payload);
var fieldCount = reader.ReadArrayHeader();
var snapshot = new CultPersistedStoreSnapshot();
@@ -217,6 +223,24 @@ public static class CultDocumentMessagePackSerialization
return snapshot;
}
+ private static void RequireOneArray(byte[] bytes)
+ {
+ var whole = new MessagePackReader(bytes);
+ try
+ {
+ if (whole.NextMessagePackType != MessagePackType.Array)
+ throw new NotSupportedException($"The store is a MessagePack {whole.NextMessagePackType}, not an array.");
+ whole.Skip();
+ }
+ catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException)
+ {
+ throw new NotSupportedException("The store is not one complete MessagePack array.", ex);
+ }
+
+ if (!whole.End)
+ throw new NotSupportedException("The store has bytes after its MessagePack array.");
+ }
+
/// Refuses a single-file snapshot this runtime cannot read, naming the format or record it found.
public static void RequireSingleFileFormat(CultPersistedStoreSnapshot snapshot)
{
@@ -241,41 +265,6 @@ public static class CultDocumentMessagePackSerialization
}
}
- ///
- /// The format header of a store file about to be replaced by a writer that does not read the store's records: null for
- /// an empty file or a legacy envelope array (an array whose first slot is not a string), the header string otherwise.
- /// A file is a store only when it is exactly one complete MessagePack array: a truncated file, bytes after the array, a
- /// value that is not an array, or a first slot that is a string but not a header this runtime reads are all refused,
- /// never taken for a legacy file, so a writer cannot overwrite what it cannot see.
- ///
- public static string? ReadStoreHeader(byte[] bytes)
- {
- if (bytes.Length == 0)
- return null;
- var whole = new MessagePackReader(bytes);
- try
- {
- if (whole.NextMessagePackType != MessagePackType.Array)
- throw new NotSupportedException($"The store is a MessagePack {whole.NextMessagePackType}, not an array; it is not rewritten.");
- whole.Skip();
- }
- catch (Exception ex) when (ex is EndOfStreamException or MessagePackSerializationException or InvalidOperationException)
- {
- throw new NotSupportedException("The store is not one complete MessagePack array; it is not rewritten.", ex);
- }
-
- if (!whole.End)
- throw new NotSupportedException("The store has bytes after its MessagePack array; it is not rewritten.");
-
- var reader = new MessagePackReader(bytes);
- if (reader.ReadArrayHeader() == 0 || reader.NextMessagePackType != MessagePackType.String)
- return null;
- // A first slot that is a string is a header, and one this runtime does not read is refused, whatever the string says.
- var header = reader.ReadString() ?? string.Empty;
- RequireSingleFileFormat(new CultPersistedStoreSnapshot { FormatVersion = header });
- return header;
- }
-
private static void WritePersistedRecord(ref MessagePackWriter writer, CultPersistedRecord record)
{
writer.WriteArrayHeader(record.Variant == null ? PersistedRecordFieldCount : VariantRecordFieldCount);
@@ -568,8 +557,6 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
return snapshot;
}
- protected override string? ReadStoreHeader(byte[] data) => CultDocumentMessagePackSerialization.ReadStoreHeader(data);
-
protected override byte[] SerializePayload(object document)
{
return CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), Registry);
diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md
index f998b8ea..d887e44e 100644
--- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md
+++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md
@@ -219,12 +219,26 @@ A reader refuses any `cultcache.store.*` header it does not know and any record
slots than its header allows, naming the version or the record's key and schema id. It
never skips a slot it does not understand.
-The headers are `v1` (no variant, no element ids), `v2` (holds a variant) and `v3` (can
-hold element ids, variants or not). A single-file store declares `v3` when any record it
-holds is of a type that reaches an object list, and a file already declared `v3` stays
-`v3` when a writer merges onto it. A directory store's manifest says
-`cultcache.store.v5.directory-content-addressed-pages` under the same rule, and
-`v4` otherwise.
+The headers are `v1` (no variant, no element ids), `v2` (holds a variant) and `v3` (holds
+an element id, variants or not). A single-file store declares `v3` when any record it
+holds carries an element id: a non-empty `[CultElementId]` value in a record's document,
+or in a variant's override values. A store that cannot read a variant's override values
+(no codec attached) counts the variant as holding one. A whole-store writer decides by
+what the store holds now, so the file drops back to `v1` (or `v2` with a variant) when
+the last id is taken away; a writer that touches only part of the file cannot see the
+rest, so a file already declared `v3` stays `v3` when it merges onto it. A directory
+store's manifest says `cultcache.store.v5.directory-content-addressed-pages` under the
+same rule, and `v4` otherwise.
+
+A single-file store is exactly one MessagePack array. Every runtime refuses a file that is
+truncated, has bytes after the array (a second store appended included), is not an array, or
+whose first slot is not a header it reads; it refuses these on open and on rewrite alike. A
+zero-byte file is an empty store. A flush or a commit replaces a file only when the
+runtime's own reader opens it, so one verdict covers open, flush and commit, and a file the
+runtime cannot read is left as it is. Rust, TypeScript and Python also open the legacy
+envelope array (an array of `key`/`type`/`payload`/`storedAt` maps) and replace it; C# has
+no legacy reader and refuses it. The shared byte vectors and each runtime's verdict are
+`tests/vectors/document-variants-c2a/readability/manifest.txt`.
## Variant Records
@@ -245,8 +259,8 @@ fifth slot:
`elementId` is `""` unless the step enters an object-list element. `value` is the
member's own MessagePack encoding, inline. A top-level Set has one step and `id` `""`;
a reader that resolves only that shape refuses the others, naming the variant.
-- A store holding at least one variant declares `cultcache.store.v2` (`v3` when it can
- hold element ids); a store holding none declares `cultcache.store.v1` (or `v3`) and is
+- A store holding at least one variant declares `cultcache.store.v2` (`v3` when it holds an
+ element id); a store holding none declares `cultcache.store.v1` (or `v3`) and is
byte-identical to what a runtime without variants writes. A `v1` header over a variant
record is refused.
- A variant's resolved document is derived in memory when the cache admits the record
@@ -276,7 +290,7 @@ elements retargets nothing.
written before ids existed loads, and its ids are minted from SHA-256 of the record
key and the element's path, so every reader mints the same ids; they reach disk with
the record's next write, or `CultCache.MintElementIds()`.
-- A store that can hold ids declares the `v3` header (above), so a reader older than ids
+- A store that holds an id declares the `v3` header (above), so a reader older than ids
refuses it instead of skipping the id slot and rewriting the elements without ids.
TypeScript, Python and Rust read `v3` and keep a `v3` header they found when they
rewrite a store; they do not decide when a document carries ids.
diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs
index f66cde48..e12ee0b9 100644
--- a/src/GameCult.Caching/CultCache.cs
+++ b/src/GameCult.Caching/CultCache.cs
@@ -3416,15 +3416,13 @@ namespace GameCult.Caching
protected FileInfo FileInfo { get; }
protected abstract byte[] SerializeSnapshot(CultPersistedStoreSnapshot snapshot);
- protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data);
///
- /// The format header of a non-empty store file this store is about to replace: null for a legacy file that carries none,
- /// the header otherwise. It throws for bytes that are not exactly one store this runtime reads, so a rewrite never
- /// overwrites a file it cannot see. The default reads the whole snapshot; a codec that can read the header alone
- /// overrides it.
+ /// This store's one reader of a non-empty store file, and so its one verdict on whether the file may be replaced: open,
+ /// flush and commit all ask it. It must throw for anything that is not exactly one store this runtime reads (a
+ /// truncated file, bytes after the store, a format it does not know), so a rewrite never overwrites a file it cannot see.
///
- protected virtual string? ReadStoreHeader(byte[] data) => DeserializeSnapshot(data).FormatVersion;
+ protected abstract CultPersistedStoreSnapshot DeserializeSnapshot(byte[] data);
protected abstract byte[] SerializePayload(object document);
protected abstract object DeserializePayload(Type documentType, byte[] payload);
@@ -3502,8 +3500,7 @@ namespace GameCult.Caching
{
using (AcquireLock(wait: true))
{
- if (ReadDisk() is { Length: > 0 } onDisk)
- ReadStoreHeader(onDisk);
+ ReadSnapshot();
WriteSnapshot(
Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)),
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()),
@@ -3529,7 +3526,7 @@ namespace GameCult.Caching
if (fileLock == null)
return CultCommitOutcome.Contended;
- var disk = ReadSnapshot(rewriting: true) ?? new CultPersistedStoreSnapshot();
+ var disk = ReadSnapshot() ?? new CultPersistedStoreSnapshot();
if (!request.ConditionsHold(disk.Records, Entries.Values))
return CultCommitOutcome.Mismatch;
@@ -3598,15 +3595,12 @@ namespace GameCult.Caching
Judging!(arriving, departing);
}
- // A commit rewrites the file it read, so it reads it as a rewrite does: the header proves the bytes are one store first.
- private CultPersistedStoreSnapshot? ReadSnapshot(bool rewriting = false)
+ // Open, flush and commit read the file through this one call, so they agree on which files are stores. A file that is
+ // gone or zero bytes is an empty store: nothing to read, nothing to lose.
+ private CultPersistedStoreSnapshot? ReadSnapshot()
{
var bytes = ReadDisk();
- if (bytes == null)
- return null;
- if (rewriting && bytes.Length > 0)
- ReadStoreHeader(bytes);
- return DeserializeSnapshot(bytes);
+ return bytes == null ? null : DeserializeSnapshot(bytes);
}
private byte[]? ReadDisk()
@@ -3616,7 +3610,7 @@ namespace GameCult.Caching
return null;
try
{
- return ReadAllBytesShared(FileInfo.FullName);
+ return ReadAllBytesShared(FileInfo.FullName) is { Length: > 0 } bytes ? bytes : null;
}
catch (FileNotFoundException)
{
diff --git a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs
index ae033e54..2a2ff2f6 100644
--- a/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs
+++ b/src/GameCult.Mesh/CultMeshSingleFileDocuments.cs
@@ -241,11 +241,15 @@ namespace GameCult.Mesh
throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry));
payload ??= Array.Empty();
+ // The file being replaced is read first, by the reader every read of it uses: one this runtime cannot read refuses the
+ // write instead of being overwritten.
+ var existing = File.Exists(path) ? ReadSingleFileSnapshot(path) : null;
+
// A typed write sees the document it replaces the file with, so its ids decide. A raw payload is opaque: the writer
// cannot see whether it holds ids, so a file already marked stays marked.
var snapshot = new CultPersistedStoreSnapshot
{
- FormatVersion = CacheBackingStore.HeaderFor(holdsIds, contentKnown ? null : ExistingHeader(path), wholeStore: contentKnown, directoryStore: false),
+ FormatVersion = CacheBackingStore.HeaderFor(holdsIds, existing?.FormatVersion, wholeStore: contentKnown, directoryStore: false),
SchemaCatalog = new[] { catalogEntry },
Records = new[]
{
@@ -266,15 +270,11 @@ namespace GameCult.Mesh
WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
}
- private static string? ExistingHeader(string path)
- {
- // A file that is gone has no header; one that cannot be read refuses the write rather than being overwritten.
- return File.Exists(path) ? CultDocumentMessagePackSerialization.ReadStoreHeader(File.ReadAllBytes(path)) : null;
- }
-
private static CultPersistedStoreSnapshot ReadSingleFileSnapshot(string path)
{
var bytes = File.ReadAllBytes(path);
+ if (bytes.Length == 0)
+ return new CultPersistedStoreSnapshot();
CultPersistedStoreSnapshot snapshot;
try
{
diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs
index 2016b975..dae82cee 100644
--- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs
+++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs
@@ -83,6 +83,10 @@ namespace GameCult.Caching.Tests
Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded { Id = id } } } }), Is.True, "a list of lists");
Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "a list under an unset element");
Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest { Id = id } } }), Is.True, "an element");
+ Assert.That(CultElementIds.Holds(new HAll { Dict = { ["a"] = new HHasIds { Items = { new EIded() } }, ["b"] = new HHasIds { Items = { new EIded { Id = id } } } } }), Is.True, "the second dictionary value");
+ Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded() }, new List { new EIded { Id = id } } } }), Is.True, "the second list");
+ Assert.That(CultElementIds.Holds(new HAll { Nest = { new ENest(), new ENest { Sub = { new EIded { Id = id } } } } }), Is.True, "the second item of a list of elements");
+ Assert.That(CultElementIds.Holds(new HAll { Grid = { new List { new EIded(), new EIded { Id = id } } } }), Is.True, "the second element of a list");
}
[Test]
diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs
index fb941453..37d938e3 100644
--- a/tests/GameCult.Caching.Tests/ElementIdTests.cs
+++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs
@@ -785,66 +785,6 @@ namespace GameCult.Caching.Tests
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "read without one, it cannot be seen, so it is marked");
}
- // A flush replaces the whole file, but only a file that is exactly one store this runtime reads (Rust, TypeScript and Python
- // refuse the same): anything else throws and is left as it was. An empty file and a legacy envelope array (an array whose
- // first slot is not a string, empty included) carry no header and are replaced.
- [TestCase("truncated", false)]
- [TestCase("v9", false)]
- [TestCase("trailing", false)]
- [TestCase("scalar", false)]
- [TestCase("map", false)]
- [TestCase("string-first", false)]
- [TestCase("empty", true)]
- [TestCase("legacy", true)]
- [TestCase("empty-array", true)]
- public void AFlushRefusesAFileThatIsNotExactlyOneStoreAndLeavesItUntouched(string file, bool replaced)
- {
- var path = PathOf($"flush-over-{file}.cc");
- using var cache = Open(path);
- cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d")));
- var bytes = NotAStore(file, File.ReadAllBytes(path));
- File.WriteAllBytes(path, bytes);
-
- if (replaced)
- {
- cache.BackingStores[0].PushAll();
- Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the file is this store's view now");
- return;
- }
-
- Assert.Throws(() => cache.BackingStores[0].PushAll());
- Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "the file was rewritten");
- }
-
- // An unconditional commit rewrites the file it read: a store with bytes after its array is refused, not rewritten.
- [Test]
- public void AnUnconditionalCommitRefusesAStoreWithBytesAfterItsArray()
- {
- var path = PathOf("commit-over-trailing.cc");
- using var cache = Open(path);
- cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d")));
- var bytes = NotAStore("trailing", File.ReadAllBytes(path));
- File.WriteAllBytes(path, bytes);
-
- Assert.Throws(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("e"), new CultRecordKey("e"))));
- Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "the file was rewritten");
- }
-
- // What a store file is not, built from a whole store file.
- private static byte[] NotAStore(string file, byte[] whole) => file switch
- {
- "truncated" => whole[..^1],
- "v9" => MessagePackSerializer.Serialize(new object[] { "cultcache.store.v9", Array.Empty(), Array.Empty() }),
- "trailing" => whole.Concat(new byte[] { 1, 2, 3 }).ToArray(),
- "scalar" => new byte[] { 0x01 },
- "map" => new byte[] { 0x80 },
- "string-first" => MessagePackSerializer.Serialize(new object[] { "hello", 1 }),
- "empty" => Array.Empty(),
- "empty-array" => new byte[] { 0x90 },
- "legacy" => MessagePackSerializer.Serialize(new object[] { new Dictionary { ["key"] = "old", ["type"] = "old", ["storedAt"] = "then" } }),
- _ => throw new ArgumentOutOfRangeException(nameof(file))
- };
-
[Test]
public void ADirectoryStoreIsMarkedByTheIdsItWritesAndAnEmptyDeckDoesNotMarkIt()
{
diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs
new file mode 100644
index 00000000..9f2cc074
--- /dev/null
+++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs
@@ -0,0 +1,115 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using GameCult.Caching.MessagePack;
+using NUnit.Framework;
+using static GameCult.Caching.Tests.ElementIdTests;
+
+namespace GameCult.Caching.Tests
+{
+ // A file is replaced by a flush or a commit exactly when this runtime's own reader opens it: one verdict per file, asked by
+ // open, flush and commit alike. The bytes and the verdict of every runtime are shared: tests/vectors/document-variants-c2a/readability.
+ public class StoreReadabilityTests
+ {
+ private const int CSharp = 1;
+ private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(IdDeck) });
+ private string _directory = "";
+
+ [SetUp]
+ public void SetUp()
+ {
+ _directory = Path.Combine(Path.GetTempPath(), "cultlib-readability-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(_directory);
+ }
+
+ [TearDown]
+ public void TearDown()
+ {
+ if (Directory.Exists(_directory))
+ Directory.Delete(_directory, true);
+ }
+
+ private static string VectorRoot()
+ {
+ for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent)
+ {
+ var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a", "readability");
+ if (Directory.Exists(candidate))
+ return candidate;
+ }
+
+ throw new DirectoryNotFoundException($"Shared readability vectors not found above {TestContext.CurrentContext.TestDirectory}.");
+ }
+
+ public static IEnumerable Vectors() =>
+ File.ReadAllLines(Path.Combine(VectorRoot(), "manifest.txt"))
+ .Where(line => line.Length > 0 && line[0] != '#')
+ .Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries))
+ .Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}"));
+
+ private static string Header(string path) => CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion;
+
+ private string Seed(string name)
+ {
+ var path = Path.Combine(_directory, name);
+ using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
+ cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "d" }, new CultRecordKey("d")));
+ return path;
+ }
+
+ [TestCaseSource(nameof(Vectors))]
+ public void OpenReadsExactlyTheFilesTheVectorsSayItReads(string vector, bool reads)
+ {
+ var path = Path.Combine(_directory, "open.cc");
+ File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), vector)));
+
+ using var cache = new CultCache();
+ cache.AddBackingStore(new SingleFileMessagePackBackingStore(path));
+ if (reads)
+ Assert.DoesNotThrow(() => cache.PullAllBackingStoresAsync().GetAwaiter().GetResult());
+ else
+ Assert.That(() => cache.PullAllBackingStoresAsync().GetAwaiter().GetResult(), Throws.Exception);
+ }
+
+ [TestCaseSource(nameof(Vectors))]
+ public void AFlushReplacesAFileExactlyWhenItReads(string vector, bool reads)
+ {
+ var path = Seed("flush.cc");
+ using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
+ var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
+ File.WriteAllBytes(path, bytes);
+ var store = cache.BackingStores[0];
+
+ if (reads)
+ {
+ store.PushAll();
+ Assert.That(Header(path), Does.StartWith("cultcache.store.v"), "the file is a store this runtime wrote");
+ return;
+ }
+
+ Assert.That(() => store.PushAll(), Throws.Exception);
+ Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten");
+ }
+
+ [TestCaseSource(nameof(Vectors))]
+ public void AnUnconditionalCommitReplacesAFileExactlyWhenItReads(string vector, bool reads)
+ {
+ var path = Seed("commit.cc");
+ using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
+ var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
+ File.WriteAllBytes(path, bytes);
+
+ if (reads)
+ {
+ cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e")));
+ Assert.That(Header(path), Does.StartWith("cultcache.store.v"), "the file is a store this runtime wrote");
+ return;
+ }
+
+ Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.Exception);
+ Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten");
+ }
+ }
+}
diff --git a/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs
new file mode 100644
index 00000000..d9cc7ef7
--- /dev/null
+++ b/tests/GameCult.Mesh.Tests/CultMeshSingleFileReadabilityTests.cs
@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using FluentAssertions;
+using GameCult.Caching;
+using GameCult.Caching.MessagePack;
+using NUnit.Framework;
+
+#nullable enable
+
+namespace GameCult.Mesh.Tests;
+
+// A single-file write replaces a file exactly when the runtime's own reader opens it, as a cache's flush does. The bytes and
+// the verdict are shared with every runtime: tests/vectors/document-variants-c2a/readability.
+public sealed class CultMeshSingleFileReadabilityTests
+{
+ private const int CSharp = 1;
+ private string _root = null!;
+
+ [SetUp]
+ public void SetUp()
+ {
+ _root = Path.Combine(Path.GetTempPath(), "cultmesh-single-file-readability", Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(_root);
+ }
+
+ [TearDown]
+ public void TearDown()
+ {
+ if (Directory.Exists(_root)) Directory.Delete(_root, true);
+ }
+
+ private static string VectorRoot()
+ {
+ for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent)
+ {
+ var candidate = Path.Combine(dir.FullName, "tests", "vectors", "document-variants-c2a", "readability");
+ if (Directory.Exists(candidate)) return candidate;
+ }
+
+ throw new DirectoryNotFoundException($"Shared readability vectors not found above {TestContext.CurrentContext.TestDirectory}.");
+ }
+
+ public static IEnumerable Vectors() =>
+ File.ReadAllLines(Path.Combine(VectorRoot(), "manifest.txt"))
+ .Where(line => line.Length > 0 && line[0] != '#')
+ .Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries))
+ .Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}"));
+
+ private static CultMeshBodyPublicationDocument Publication() => new()
+ {
+ BodyId = "aetheria:entities",
+ ProducerId = "aetheria",
+ SchemaId = "eve.entity_soa.v1"
+ };
+
+ [TestCaseSource(nameof(Vectors))]
+ public void ATypedWriteReplacesAFileExactlyWhenItReads(string vector, bool reads)
+ {
+ var path = Path.Combine(_root, "doc.cc");
+ var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
+ File.WriteAllBytes(path, bytes);
+
+ if (reads)
+ {
+ CultMesh.WriteSingleFileDocument(path, new CultRecordKey("publication"), Publication());
+ CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Should().ContainSingle();
+ return;
+ }
+
+ Assert.That(() => CultMesh.WriteSingleFileDocument(path, new CultRecordKey("publication"), Publication()), Throws.Exception);
+ File.ReadAllBytes(path).Should().Equal(bytes, "a file this runtime cannot read was rewritten");
+ }
+}
diff --git a/tests/vectors/document-variants-c2a/readability/.gitattributes b/tests/vectors/document-variants-c2a/readability/.gitattributes
new file mode 100644
index 00000000..291377b6
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/.gitattributes
@@ -0,0 +1 @@
+*.bin binary
diff --git a/tests/vectors/document-variants-c2a/readability/empty-array.bin b/tests/vectors/document-variants-c2a/readability/empty-array.bin
new file mode 100644
index 00000000..9e99dd5e
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/empty-array.bin
@@ -0,0 +1 @@
+�
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/generate.py b/tests/vectors/document-variants-c2a/readability/generate.py
new file mode 100644
index 00000000..bb63d0a9
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/generate.py
@@ -0,0 +1,33 @@
+"""Generates the shared store-readability vectors. Every runtime's tests read the committed bytes and the verdicts in
+manifest.txt; nothing regenerates them at test time. Run: python generate.py (needs msgpack).
+
+A file may be replaced by a flush or a commit exactly when the runtime's own reader opens it, so one verdict per runtime
+covers open, flush and commit. Each vector is the bytes named; the trailing and truncated ones are cut from v3-base.
+"""
+import pathlib
+
+import msgpack
+
+here = pathlib.Path(__file__).resolve().parent
+whole = (here.parent / "v3-base.msgpack").read_bytes()
+pack = lambda value: msgpack.packb(value, use_bin_type=True)
+
+vectors = {
+ "zero-byte": b"",
+ "empty-array": pack([]),
+ # A legacy envelope array: an array of maps. Rust, TypeScript and Python read it; C# has no legacy reader.
+ "legacy-envelopes": pack([{"key": "old", "type": "old.type", "payload": pack([]), "storedAt": "2026-01-01T00:00:00Z"}]),
+ "ints": pack([1, 2, 3]),
+ "nil-first": pack([None, "x"]),
+ "nested-array": pack([["cultcache.store.v9", [], []]]),
+ "string-first": pack(["hello", 1]),
+ "header-v9": pack(["cultcache.store.v9", [], []]),
+ "scalar": b"\x01",
+ "map": b"\x80",
+ "truncated": whole[:-1],
+ "trailing-bytes": whole + b"\x01\x02\x03",
+ "trailing-store": whole + whole,
+}
+for name, data in vectors.items():
+ (here / f"{name}.bin").write_bytes(data)
+print(len(vectors), "vectors")
diff --git a/tests/vectors/document-variants-c2a/readability/header-v9.bin b/tests/vectors/document-variants-c2a/readability/header-v9.bin
new file mode 100644
index 00000000..42f9eba0
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/header-v9.bin
@@ -0,0 +1 @@
+“²cultcache.store.v9��
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/ints.bin b/tests/vectors/document-variants-c2a/readability/ints.bin
new file mode 100644
index 00000000..dfedbc05
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/ints.bin
@@ -0,0 +1 @@
+“
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin b/tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin
new file mode 100644
index 00000000..a55b231f
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/legacy-envelopes.bin
@@ -0,0 +1 @@
+‘„£key£old¤type¨old.type§payloadÄ�¨storedAt´2026-01-01T00:00:00Z
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/manifest.txt b/tests/vectors/document-variants-c2a/readability/manifest.txt
new file mode 100644
index 00000000..55e028f9
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/manifest.txt
@@ -0,0 +1,16 @@
+# Whether each runtime's own reader opens the file. A flush and a commit replace a file exactly when it reads.
+# Columns: file, csharp, rust, typescript, python. `../v3-base.msgpack` is the one valid store.
+../v3-base.msgpack reads reads reads reads
+zero-byte.bin reads reads reads reads
+empty-array.bin reads reads reads reads
+legacy-envelopes.bin refuses reads reads reads
+ints.bin refuses refuses refuses refuses
+nil-first.bin refuses refuses refuses refuses
+nested-array.bin refuses refuses refuses refuses
+string-first.bin refuses refuses refuses refuses
+header-v9.bin refuses refuses refuses refuses
+scalar.bin refuses refuses refuses refuses
+map.bin refuses refuses refuses refuses
+truncated.bin refuses refuses refuses refuses
+trailing-bytes.bin refuses refuses refuses refuses
+trailing-store.bin refuses refuses refuses refuses
diff --git a/tests/vectors/document-variants-c2a/readability/map.bin b/tests/vectors/document-variants-c2a/readability/map.bin
new file mode 100644
index 00000000..5416677b
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/map.bin
@@ -0,0 +1 @@
+€
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/nested-array.bin b/tests/vectors/document-variants-c2a/readability/nested-array.bin
new file mode 100644
index 00000000..34b09870
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/nested-array.bin
@@ -0,0 +1 @@
+‘“²cultcache.store.v9��
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/nil-first.bin b/tests/vectors/document-variants-c2a/readability/nil-first.bin
new file mode 100644
index 00000000..5ce298f2
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/nil-first.bin
@@ -0,0 +1 @@
+’À¡x
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/scalar.bin b/tests/vectors/document-variants-c2a/readability/scalar.bin
new file mode 100644
index 00000000..6b2aaa76
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/scalar.bin
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/string-first.bin b/tests/vectors/document-variants-c2a/readability/string-first.bin
new file mode 100644
index 00000000..862a69f7
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/string-first.bin
@@ -0,0 +1 @@
+’¥hello
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/trailing-bytes.bin b/tests/vectors/document-variants-c2a/readability/trailing-bytes.bin
new file mode 100644
index 00000000..d191fcfe
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/trailing-bytes.bin
@@ -0,0 +1 @@
+“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/trailing-store.bin b/tests/vectors/document-variants-c2a/readability/trailing-store.bin
new file mode 100644
index 00000000..2d3ca7ec
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/trailing-store.bin
@@ -0,0 +1 @@
+“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/truncated.bin b/tests/vectors/document-variants-c2a/readability/truncated.bin
new file mode 100644
index 00000000..96796824
--- /dev/null
+++ b/tests/vectors/document-variants-c2a/readability/truncated.bin
@@ -0,0 +1 @@
+“²cultcache.store.v3‘—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta
\ No newline at end of file
diff --git a/tests/vectors/document-variants-c2a/readability/zero-byte.bin b/tests/vectors/document-variants-c2a/readability/zero-byte.bin
new file mode 100644
index 00000000..e69de29b