diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 0ad56f70..3c6d61c6 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -4,6 +4,7 @@ import base64 import json import os import threading +from dataclasses import replace from pathlib import Path from typing import Any @@ -172,7 +173,16 @@ def _encode_snapshot(envelopes: list[CultCacheEnvelope], format_version: str) -> if supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids) else _default_catalog_entry(envelope) ) - catalog_by_schema_id.setdefault(published.schema_id, published) + # Entries that share an id are one schema seen by different writers: the entry written lists every id any of them lists. + existing = catalog_by_schema_id.get(published.schema_id) + catalog_by_schema_id[published.schema_id] = ( + published + if existing is None + else replace( + existing, + compatible_schema_ids=tuple(dict.fromkeys((*existing.compatible_schema_ids, *published.compatible_schema_ids))), + ) + ) catalog = [ _encode_catalog_entry(entry) @@ -200,12 +210,14 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None if not isinstance(decoded[1], list) or not isinstance(decoded[2], list): raise ValueError("CultCache v1 snapshot must contain a schema catalog and record array") + # An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible. + entries = [_decode_catalog_entry(raw_entry) for raw_entry in decoded[1]] catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {} - for raw_entry in decoded[1]: - entry = _decode_catalog_entry(raw_entry) - catalog_by_schema_id[entry.schema_id] = entry + for entry in entries: + catalog_by_schema_id.setdefault(entry.schema_id, entry) + for entry in entries: for compatible_schema_id in entry.compatible_schema_ids: - catalog_by_schema_id[compatible_schema_id] = entry + catalog_by_schema_id.setdefault(compatible_schema_id, entry) envelopes: list[CultCacheEnvelope] = [] for raw_record in decoded[2]: diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 2099acb1..4596c5ab 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -315,6 +315,41 @@ class CultCacheTests(unittest.TestCase): self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"]) self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1) + # Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists, + # whichever order the records arrive in. + def test_the_store_writer_merges_the_compatible_ids_of_entries_that_share_a_schema_id_in_either_order(self) -> None: + import msgpack # type: ignore + from cultcache_py import CultCacheSchemaCatalogEntry + from cultcache_py.stores import _encode_snapshot + + def entry(compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry: + return CultCacheSchemaCatalogEntry( + schema_id="tests.merge.x", schema_name="tests.merge", schema_version="tests.merge.v1", + content_hash="tests.merge.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(), + ) + + def record(key: str, schema_id: str, compatible: tuple[str, ...]) -> CultCacheEnvelope: + return CultCacheEnvelope( + key=key, type="tests.merge", payload=msgpack.packb({"key": key}, use_bin_type=True), + stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=entry(compatible), + ) + + a = record("a", "tests.merge.x", ("tests.merge.x",)) + b = record("b", "tests.merge.y", ("tests.merge.x", "tests.merge.y")) + for order in ([a, b], [b, a]): + catalog = _encode_snapshot(order, "cultcache.store.v1")[1] + self.assertEqual([sorted(entry[5]) for entry in catalog], [["tests.merge.x", "tests.merge.y"]]) + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all([b]) + SingleFileMessagePackBackingStore(path).push(a) + self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2) + + def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None: + store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / "own-id-over-compatible-v3.bin") + self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"]) + def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None: import msgpack # type: ignore diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index f7a63a0f..2fa649b0 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -379,7 +379,7 @@ impl StoreUnreadableError { impl std::fmt::Display for StoreUnreadableError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "failed to decode MessagePack {}: {:#}", self.path.display(), self.source) + write!(formatter, "failed to decode MessagePack {}", self.path.display()) } } @@ -2669,16 +2669,16 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result> { let snapshot: PersistedStoreSnapshot = rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?; - let catalog = snapshot - .1 - .into_iter() - .flat_map(|entry| { - // A schema is published by its id or as a compatible id. - let published: Vec = std::iter::once(entry.0).chain(entry.5).collect(); - let name = entry.1; - published.into_iter().map(move |id| (id, name.clone())) - }) - .collect::>(); + // A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible. + let mut catalog = BTreeMap::::new(); + for entry in &snapshot.1 { + catalog.entry(entry.0.clone()).or_insert_with(|| entry.1.clone()); + } + for entry in &snapshot.1 { + for compatible in &entry.5 { + catalog.entry(compatible.clone()).or_insert_with(|| entry.1.clone()); + } + } snapshot .2 .into_iter() @@ -4801,8 +4801,10 @@ mod tests { if !reads { let error = outcome.unwrap_err(); let typed = error.downcast_ref::().unwrap_or_else(|| panic!("{what}: {error:#}")); - let cause = std::error::Error::source(typed).unwrap_or_else(|| panic!("{what}: the cause is the source")); - assert!(typed.to_string().contains(&cause.to_string()), "{what}: the message carries the cause: {typed}"); + let cause = std::error::Error::source(typed).unwrap_or_else(|| panic!("{what}: the cause is the source")).to_string(); + // The message is the error's own; the cause is printed once, by the chain. + assert!(!typed.to_string().contains(&cause), "{what}: the message repeats the cause: {typed}"); + assert_eq!(format!("{error:#}").matches(&cause).count(), 1, "{what}: {error:#}"); assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a file it cannot read"); } else if operation != "open" { assert_eq!(store_header(&std::fs::read(&path)?)?.as_deref(), Some(header), "{what}"); @@ -4821,6 +4823,22 @@ mod tests { Ok(()) } + // An id one entry owns and a later entry lists as compatible names the entry that owns it. + #[test] + fn an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + std::fs::copy( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/vectors/document-variants-c2a/readability/own-id-over-compatible-v3.bin"), + &path, + )?; + let envelopes = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + assert_eq!(envelopes.len(), 2); + assert!(envelopes.iter().all(|envelope| envelope.r#type == "vectors.item"), "{envelopes:?}"); + Ok(()) + } + // A store cut short is refused whatever its version, and left as it is. #[test] fn a_truncated_v1_or_v2_store_is_refused_on_open_and_rewrite() -> Result<()> { diff --git a/packages/cultcache-ts/src/cult-cache-inspector.ts b/packages/cultcache-ts/src/cult-cache-inspector.ts index 09eb9ef1..ad684859 100644 --- a/packages/cultcache-ts/src/cult-cache-inspector.ts +++ b/packages/cultcache-ts/src/cult-cache-inspector.ts @@ -68,11 +68,18 @@ function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: [St } const catalog = catalogRaw.map(decodeCatalogEntry); + // An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible. const catalogBySchemaId = new Map(); for (const entry of catalog) { - catalogBySchemaId.set(entry.schemaId, entry); + if (!catalogBySchemaId.has(entry.schemaId)) { + catalogBySchemaId.set(entry.schemaId, entry); + } + } + for (const entry of catalog) { for (const compatibleSchemaId of entry.compatibleSchemaIds) { - catalogBySchemaId.set(compatibleSchemaId, entry); + if (!catalogBySchemaId.has(compatibleSchemaId)) { + catalogBySchemaId.set(compatibleSchemaId, entry); + } } } const records = recordsRaw.map((record) => decodeV1Record(record, catalogBySchemaId)); diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 263c5764..46af9bbc 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -239,9 +239,19 @@ function catalogEntriesFor(entries: CultCacheEnvelope[]): Map(); - for (const entry of catalogRaw) { - const catalogEntry = decodeCatalogEntry(entry); - catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry); + for (const catalogEntry of catalogEntries) { + if (!catalogBySchemaId.has(catalogEntry.schemaId)) { + catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry); + } + } + for (const catalogEntry of catalogEntries) { for (const compatibleSchemaId of catalogEntry.compatibleSchemaIds ?? []) { - catalogBySchemaId.set(compatibleSchemaId, catalogEntry); + if (!catalogBySchemaId.has(compatibleSchemaId)) { + catalogBySchemaId.set(compatibleSchemaId, catalogEntry); + } } } diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 2e865443..951b3896 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -1344,6 +1344,61 @@ test("the store publishes a record's schema id even when the envelope's catalog assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 1); }); +// Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists, +// whichever order the records arrive in. +test("the store writer merges the compatible ids of entries that share a schema id, in either order", async () => { + const entry = (compatibleSchemaIds: string[]) => ({ + schemaId: "tests.merge.x", + schemaName: "tests.merge", + schemaVersion: "tests.merge.v1", + contentHash: "tests.merge.x", + canonicalSchemaJson: "", + compatibleSchemaIds, + members: [], + }); + const record = (key: string, schemaId: string, compatibleSchemaIds: string[]) => ({ + key, + type: "tests.merge", + schemaId, + payload: encode({ key }), + storedAt: "2026-09-30T00:00:00.0000000Z", + catalogEntry: entry(compatibleSchemaIds), + }); + const a = record("a", "tests.merge.x", ["tests.merge.x"]); + const b = record("b", "tests.merge.y", ["tests.merge.x", "tests.merge.y"]); + const dir = await mkdtemp(join(tmpdir(), "cultcache-merge-")); + + for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) { + const file = join(dir, `${name}.msgpack`); + await new SingleFileMessagePackBackingStore(file).pushAll([...order]); + const decoded = decode(await readFile(file)) as [string, unknown[][], unknown[][]]; + assert.deepEqual(decoded[1].map((catalogEntry) => (catalogEntry[5] as string[]).slice().sort()), [["tests.merge.x", "tests.merge.y"]], name); + assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2, name); + } + + // A put over a key of a file whose other record is stored under the wider entry's compatible id. + const file = join(dir, "push.msgpack"); + await new SingleFileMessagePackBackingStore(file).pushAll([b]); + await new SingleFileMessagePackBackingStore(file).push(a); + assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2); +}); + +test("an id an entry owns names that entry, not one that lists it as compatible", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-own-id-")); + const file = await copyVector(dir, join(c2aVectors, "readability", "own-id-over-compatible-v3.bin"), "own.msgpack"); + const envelopes = await new SingleFileMessagePackBackingStore(file).pullAll(); + assert.deepEqual(envelopes.map((envelope) => envelope.type), ["vectors.item", "vectors.item"]); + const inspection = inspectCultCacheBytes("own.msgpack", await readFile(file)); + assert.deepEqual(inspection.records.map((record) => record.schemaName), ["vectors.item", "vectors.item"]); +}); + +// The inspector reads what the reader reads: a record under an id its catalog lists only as compatible is inspected. +test("CultCache inspector reads a record whose schema the catalog publishes only as a compatible id", async () => { + const bytes = await readFile(join(c2aVectors, "readability", "compatible-id-only-v3.bin")); + const inspection = inspectCultCacheBytes("compatible.msgpack", bytes); + assert.deepEqual(inspection.records.map((record) => record.schemaName), ["vectors.item", "vectors.item"]); +}); + test("a record loaded under a compatible schema id is written back under the id its catalog entry carries", async () => { const file = join(await mkdtemp(join(tmpdir(), "cultcache-foreign-id-")), "store.msgpack"); const payload = encode({ name: "old" }); diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 42a59f9d..bb26a8c8 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -37,11 +37,9 @@ def apply_raw_document_record( ) -> CultNetAppliedRecord: wire = record.to_wire() if isinstance(record, CultNetRawDocumentRecord) else record schema_id = str(wire["schemaId"]) - document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - wire, - ) + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, wire) + # The record is applied, stamped and reported under the local schema id, as the cache stores it and a delete reports it. + resolved_schema_id = document.catalog_entry().schema_id envelope = raw_record_to_envelope(document, resolved_schema_id, wire) value = cache.put_envelope(document, envelope) return CultNetAppliedRecord( @@ -82,11 +80,8 @@ def apply_raw_snapshot( if not isinstance(record, dict): continue schema_id = str(record["schemaId"]) - document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record( - documents_by_schema_id, - schema_id, - record, - ) + document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, record) + resolved_schema_id = document.catalog_entry().schema_id envelope = raw_record_to_envelope(document, resolved_schema_id, record) batch = batches.setdefault(document.type, (document, [])) batch[1].append((resolved_schema_id, envelope)) diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index fd12e9a4..b8aad39d 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -1818,7 +1818,8 @@ class CultNetTests(unittest.TestCase): return cache cache = open_cache() - apply_raw_document_record(cache, schema_document_map([document]), record) + applied = apply_raw_document_record(cache, schema_document_map([document]), record) + self.assertEqual(applied.schema_id, document.catalog_entry().schema_id) self.assertEqual( cache.get_required_envelope(document, "policy:1").schema_id, document.catalog_entry().schema_id ) diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 15f4423c..7063e3dc 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -251,11 +251,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentIndex[key] = ToIndexRecord(stored); } - var catalogCandidates = currentManifest.SchemaCatalog - .Concat(_durableCatalog) - .Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())) - .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) - .Select(group => group.Last()) + var catalogCandidates = CultSchemaCatalogEntry.MergeById( + currentManifest.SchemaCatalog + .Concat(_durableCatalog) + .Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())), + preferLast: true) .ToArray(); var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal); var targetCatalog = catalogCandidates @@ -464,8 +464,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) && !string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal)) { - throw new InvalidOperationException( - $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable."); + throw new CultStoreUnreadableException( + $"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable.", + _manifestFile.FullName); } return snapshot; diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 2a695857..ff753ce8 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -28,6 +28,33 @@ namespace GameCult.Caching public string[] CompatibleSchemaIds { get; set; } = Array.Empty(); public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); + + // One entry per schema id, whatever order the candidates come in: entries that share an id are one schema seen by different + // writers, so the entry written lists every id any of them lists, and its other fields are the first's (or the last's). + internal static IEnumerable MergeById(IEnumerable entries, bool preferLast = false) + { + foreach (var group in entries.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)) + { + var members = group.ToArray(); + var chosen = preferLast ? members[^1] : members[0]; + if (members.Length == 1) + { + yield return chosen; + continue; + } + + yield return new CultSchemaCatalogEntry + { + SchemaId = chosen.SchemaId, + SchemaName = chosen.SchemaName, + SchemaVersion = chosen.SchemaVersion, + ContentHash = chosen.ContentHash, + CanonicalSchemaJson = chosen.CanonicalSchemaJson, + CompatibleSchemaIds = members.SelectMany(entry => entry.CompatibleSchemaIds).Distinct(StringComparer.Ordinal).ToArray(), + Members = chosen.Members + }; + } + } } public sealed class CultSchemaMemberCatalogEntry @@ -3637,9 +3664,7 @@ namespace GameCult.Caching var snapshot = new CultPersistedStoreSnapshot { FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)), - SchemaCatalog = catalog - .GroupBy(entry => entry.SchemaId, StringComparer.Ordinal) - .Select(group => group.First()) + SchemaCatalog = CultSchemaCatalogEntry.MergeById(catalog) // An entry stays while it publishes a schema id some record carries, as its own id or a compatible one. .Where(entry => used.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(used.Contains)) .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) diff --git a/src/GameCult.Caching/GameCult.Caching.csproj b/src/GameCult.Caching/GameCult.Caching.csproj index 51253f8f..91c9133b 100644 --- a/src/GameCult.Caching/GameCult.Caching.csproj +++ b/src/GameCult.Caching/GameCult.Caching.csproj @@ -7,6 +7,7 @@ + diff --git a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs index d00630af..848ecb1b 100644 --- a/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs +++ b/tests/GameCult.Caching.Tests/PreCut2StoreFormatTests.cs @@ -127,6 +127,8 @@ namespace GameCult.Caching.Tests [TestCase("current-catalog-bad-canonical.msgpack")] [TestCase("current-catalog-bad-members.msgpack")] [TestCase("current-catalog-bad-compat.msgpack")] + [TestCase("current-catalog-bad-slot5-nil.msgpack")] + [TestCase("current-catalog-bad-slot5-int.msgpack")] public void SingleFileRefusesAMalformedCurrentCatalogEntry(string vector) { Assert.That(Refusal(vector), Is.Not.Empty); diff --git a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs index 2351f9d9..d80ee40d 100644 --- a/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs +++ b/tests/GameCult.Caching.Tests/StoreReadabilityTests.cs @@ -155,6 +155,67 @@ namespace GameCult.Caching.Tests Assert.That(rewritten.Records.Select(record => record.SchemaId), Does.Contain("vectors.old.id")); } + // An id one entry owns and a later entry lists as compatible names the entry that owns it. + [Test] + public void AnIdAnEntryOwnsNamesThatEntryNotOneThatListsItAsCompatible() + { + var path = Path.Combine(_directory, "own-id.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "own-id-over-compatible-v3.bin"))); + + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }); + Assert.That(cache.Get(new CultRecordKey("alpha")), Is.Not.Null, "the record read as the schema that owns its id"); + } + + // A directory rewrite where the entry that lists a kept record's id shares its own id with the schema being written keeps that + // id in the entry it writes, so the manifest still reads. + [Test] + public void ADirectoryRewriteMergesEntriesThatShareAnIdAndKeepsEveryIdTheyList() + { + var path = DirectoryStore("merged.cc"); + var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var entry = manifest.SchemaCatalog.Single(); + manifest.Records.Single().SchemaId = "vectors.old.id"; + entry.CompatibleSchemaIds = new[] { entry.SchemaId, "vectors.old.id" }; + File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest)); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true })) + cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))); + + var rewritten = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + Assert.That(rewritten.SchemaCatalog.Single().CompatibleSchemaIds, Does.Contain("vectors.old.id")); + } + + // Entries that share an id are merged whatever order they arrive in: every id any of them lists survives. + [Test] + public void EntriesThatShareAnIdMergeTheirCompatibleIdsInEitherOrder() + { + CultSchemaCatalogEntry Entry(params string[] compatible) => new() + { + SchemaId = "x", SchemaName = "n", SchemaVersion = "n.v1", ContentHash = "h", CompatibleSchemaIds = compatible + }; + var plain = Entry("x"); + var wide = Entry("x", "y"); + + foreach (var order in new[] { new[] { plain, wide }, new[] { wide, plain } }) + foreach (var preferLast in new[] { false, true }) + { + var merged = CultSchemaCatalogEntry.MergeById(order, preferLast).Single(); + Assert.That(merged.CompatibleSchemaIds, Is.EquivalentTo(new[] { "x", "y" })); + } + } + + // A directory manifest in a format the directory store does not read is refused with the typed exception, naming it. + [Test] + public void ADirectoryManifestInAnotherFormatIsRefusedNamingIt() + { + var path = Path.Combine(_directory, "wrong-format.cc"); + File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "..", "v3-base.msgpack"))); + + Assert.That( + () => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }), + Throws.TypeOf().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path)); + } + // A commit that lands onto the file as it is keeps the header the file carries: a store already v3 stays v3. [TestCase("../v3-base.msgpack", "cultcache.store.v3")] [TestCase("compatible-id-only-v3.bin", "cultcache.store.v3")] diff --git a/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs b/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs index 579414ee..44d71051 100644 --- a/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs +++ b/tests/GameCult.Mesh.Tests/CultMeshVariantRefusalTests.cs @@ -61,6 +61,8 @@ public sealed class CultMeshVariantRefusalTests [TestCase("current-catalog-bad-canonical.msgpack")] [TestCase("current-catalog-bad-members.msgpack")] [TestCase("current-catalog-bad-compat.msgpack")] + [TestCase("current-catalog-bad-slot5-nil.msgpack")] + [TestCase("current-catalog-bad-slot5-int.msgpack")] public void ReadSingleFileDocument_RefusesAMalformedCurrentCatalogEntry(string vector) { Refusal(vector).Path.Should().Be(VectorPath(vector)); diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-slot5-int.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-int.msgpack new file mode 100644 index 00000000..5f21bf61 Binary files /dev/null and b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-int.msgpack differ diff --git a/tests/vectors/document-variants-c0/current-catalog-bad-slot5-nil.msgpack b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-nil.msgpack new file mode 100644 index 00000000..f6fc971f Binary files /dev/null and b/tests/vectors/document-variants-c0/current-catalog-bad-slot5-nil.msgpack differ diff --git a/tests/vectors/document-variants-c0/generate.py b/tests/vectors/document-variants-c0/generate.py index 1ff1ebae..d87e3996 100644 --- a/tests/vectors/document-variants-c0/generate.py +++ b/tests/vectors/document-variants-c0/generate.py @@ -18,6 +18,10 @@ at test time. Run: python generate.py (needs msgpack). v1 header, plain records, and the first catalog entry of the current layout with one slot malformed (canonical schema json an int; members an int; a compatible id an int). Not the older layout, so CultMesh refuses them as CultCache does + current-catalog-bad-slot5-nil.msgpack + current-catalog-bad-slot5-int.msgpack + the same, with slot 5 (the compatible ids) nil or an integer: neither is a string, so neither is + the older layout v1-base.msgpack NOT generated here: written once by the Python runtime at CultLib 69a21bb (SingleFileMessagePackBackingStore.push of alpha and beta, type vectors.item, before any C0 code existed) @@ -60,6 +64,8 @@ for name, entry in ( ("current-catalog-bad-canonical", current[:4] + [5]), ("current-catalog-bad-members", current[:6] + [5]), ("current-catalog-bad-compat", current[:5] + [[7]] + current[6:]), + ("current-catalog-bad-slot5-nil", current[:5] + [None] + current[6:]), + ("current-catalog-bad-slot5-int", current[:5] + [7] + current[6:]), ): # One catalog entry and the records that use it: the older-layout reader takes an entry's slot 5 for a string, so a second # current-layout entry would make it refuse for that reason, not for the malformed slot under test. diff --git a/tests/vectors/document-variants-c2a/readability/generate.py b/tests/vectors/document-variants-c2a/readability/generate.py index ae7433b9..b9932c7d 100644 --- a/tests/vectors/document-variants-c2a/readability/generate.py +++ b/tests/vectors/document-variants-c2a/readability/generate.py @@ -41,6 +41,8 @@ vectors = { "missing-schema-versioned-v3": pack([base[0], [], [record[:3] + [pack(["vectors.item.v1", "n", 1])] for record in base[2]]]), # The record names its schema by an id the catalog publishes only as a compatible id of an entry with another id. "compatible-id-only-v3": pack([base[0], [[base[1][0][0] + ".next"] + base[1][0][1:]], base[2]]), + # An id that one entry owns and a later entry lists as compatible names the entry that owns it (here, vectors.item). + "own-id-over-compatible-v3": pack([base[0], base[1] + [["vectors.item.next", "vectors.other", "vectors.other.v1", "vectors.item.next", "{}", ["vectors.item"], []]], base[2]]), # v3-base plus a variant of alpha: a fifth record slot, [baseKey, overrides[]], over an empty payload, overriding the name. Only C# reads it. "variant-slot-v3": pack([base[0], base[1], base[2] + [["gamma"] + base[2][0][1:3] + [b"", ["alpha", [[0, [[0, ""]], "", "gamma"]]]]]]), # v3-base with a fourth top-level slot after the records. diff --git a/tests/vectors/document-variants-c2a/readability/manifest.txt b/tests/vectors/document-variants-c2a/readability/manifest.txt index aac97e19..841c4ec4 100644 --- a/tests/vectors/document-variants-c2a/readability/manifest.txt +++ b/tests/vectors/document-variants-c2a/readability/manifest.txt @@ -24,3 +24,4 @@ missing-schema-v3.bin refuses refuses refuses refuses variant-slot-v3.bin reads refuses refuses refuses missing-schema-versioned-v3.bin refuses refuses refuses refuses compatible-id-only-v3.bin reads reads reads reads +own-id-over-compatible-v3.bin reads reads reads reads diff --git a/tests/vectors/document-variants-c2a/readability/own-id-over-compatible-v3.bin b/tests/vectors/document-variants-c2a/readability/own-id-over-compatible-v3.bin new file mode 100644 index 00000000..66221428 --- /dev/null +++ b/tests/vectors/document-variants-c2a/readability/own-id-over-compatible-v3.bin @@ -0,0 +1 @@ +“²cultcache.store.v3’—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item�—±vectors.item.next­vectors.other°vectors.other.v1±vectors.item.next¢{}‘¬vectors.item�’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta \ No newline at end of file