Catalog writers merge entries that share an id; every reader resolves an id by its owner first
Entries that share a schema id are one schema seen by different writers, so the writer lists every id any of them lists, in any order: C# single-file and directory (CultSchemaCatalogEntry.MergeById), TypeScript and Python. Rust synthesises one entry per id and has no such merge. An id that one entry owns and another lists as compatible names the owner in Rust, TypeScript, Python and the TS inspector, as C# already did (own-id-over-compatible-v3). Also: a directory manifest in an unreadable format is the typed refusal naming it, cultnet-py applies and reports a record under the local schema id, and Rust's error prints its cause once. Vectors for the Mesh layout discriminator: a nil and an integer at catalog slot 5. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens
parent
2d2fad1f45
commit
f96a0e7f7a
20 changed files with 290 additions and 48 deletions
|
|
@ -4,6 +4,7 @@ import base64
|
|||
import json
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -172,7 +173,16 @@ def _encode_snapshot(envelopes: list[CultCacheEnvelope], format_version: str) ->
|
|||
if supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids)
|
||||
else _default_catalog_entry(envelope)
|
||||
)
|
||||
catalog_by_schema_id.setdefault(published.schema_id, published)
|
||||
# Entries that share an id are one schema seen by different writers: the entry written lists every id any of them lists.
|
||||
existing = catalog_by_schema_id.get(published.schema_id)
|
||||
catalog_by_schema_id[published.schema_id] = (
|
||||
published
|
||||
if existing is None
|
||||
else replace(
|
||||
existing,
|
||||
compatible_schema_ids=tuple(dict.fromkeys((*existing.compatible_schema_ids, *published.compatible_schema_ids))),
|
||||
)
|
||||
)
|
||||
|
||||
catalog = [
|
||||
_encode_catalog_entry(entry)
|
||||
|
|
@ -200,12 +210,14 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None
|
|||
if not isinstance(decoded[1], list) or not isinstance(decoded[2], list):
|
||||
raise ValueError("CultCache v1 snapshot must contain a schema catalog and record array")
|
||||
|
||||
# An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible.
|
||||
entries = [_decode_catalog_entry(raw_entry) for raw_entry in decoded[1]]
|
||||
catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {}
|
||||
for raw_entry in decoded[1]:
|
||||
entry = _decode_catalog_entry(raw_entry)
|
||||
catalog_by_schema_id[entry.schema_id] = entry
|
||||
for entry in entries:
|
||||
catalog_by_schema_id.setdefault(entry.schema_id, entry)
|
||||
for entry in entries:
|
||||
for compatible_schema_id in entry.compatible_schema_ids:
|
||||
catalog_by_schema_id[compatible_schema_id] = entry
|
||||
catalog_by_schema_id.setdefault(compatible_schema_id, entry)
|
||||
|
||||
envelopes: list[CultCacheEnvelope] = []
|
||||
for raw_record in decoded[2]:
|
||||
|
|
|
|||
|
|
@ -315,6 +315,41 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"])
|
||||
self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1)
|
||||
|
||||
# Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists,
|
||||
# whichever order the records arrive in.
|
||||
def test_the_store_writer_merges_the_compatible_ids_of_entries_that_share_a_schema_id_in_either_order(self) -> None:
|
||||
import msgpack # type: ignore
|
||||
from cultcache_py import CultCacheSchemaCatalogEntry
|
||||
from cultcache_py.stores import _encode_snapshot
|
||||
|
||||
def entry(compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry:
|
||||
return CultCacheSchemaCatalogEntry(
|
||||
schema_id="tests.merge.x", schema_name="tests.merge", schema_version="tests.merge.v1",
|
||||
content_hash="tests.merge.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(),
|
||||
)
|
||||
|
||||
def record(key: str, schema_id: str, compatible: tuple[str, ...]) -> CultCacheEnvelope:
|
||||
return CultCacheEnvelope(
|
||||
key=key, type="tests.merge", payload=msgpack.packb({"key": key}, use_bin_type=True),
|
||||
stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=entry(compatible),
|
||||
)
|
||||
|
||||
a = record("a", "tests.merge.x", ("tests.merge.x",))
|
||||
b = record("b", "tests.merge.y", ("tests.merge.x", "tests.merge.y"))
|
||||
for order in ([a, b], [b, a]):
|
||||
catalog = _encode_snapshot(order, "cultcache.store.v1")[1]
|
||||
self.assertEqual([sorted(entry[5]) for entry in catalog], [["tests.merge.x", "tests.merge.y"]])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "store.msgpack"
|
||||
SingleFileMessagePackBackingStore(path).push_all([b])
|
||||
SingleFileMessagePackBackingStore(path).push(a)
|
||||
self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2)
|
||||
|
||||
def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None:
|
||||
store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / "own-id-over-compatible-v3.bin")
|
||||
self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"])
|
||||
|
||||
def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None:
|
||||
import msgpack # type: ignore
|
||||
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ impl StoreUnreadableError {
|
|||
|
||||
impl std::fmt::Display for StoreUnreadableError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(formatter, "failed to decode MessagePack {}: {:#}", self.path.display(), self.source)
|
||||
write!(formatter, "failed to decode MessagePack {}", self.path.display())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2669,16 +2669,16 @@ fn decode_store_snapshot(bytes: &[u8]) -> Result<Vec<CultCacheEnvelope>> {
|
|||
let snapshot: PersistedStoreSnapshot =
|
||||
rmp_serde::from_slice(bytes).context("failed to decode CultCache v1 snapshot")?;
|
||||
|
||||
let catalog = snapshot
|
||||
.1
|
||||
.into_iter()
|
||||
.flat_map(|entry| {
|
||||
// A schema is published by its id or as a compatible id.
|
||||
let published: Vec<String> = std::iter::once(entry.0).chain(entry.5).collect();
|
||||
let name = entry.1;
|
||||
published.into_iter().map(move |id| (id, name.clone()))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
// A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible.
|
||||
let mut catalog = BTreeMap::<String, String>::new();
|
||||
for entry in &snapshot.1 {
|
||||
catalog.entry(entry.0.clone()).or_insert_with(|| entry.1.clone());
|
||||
}
|
||||
for entry in &snapshot.1 {
|
||||
for compatible in &entry.5 {
|
||||
catalog.entry(compatible.clone()).or_insert_with(|| entry.1.clone());
|
||||
}
|
||||
}
|
||||
snapshot
|
||||
.2
|
||||
.into_iter()
|
||||
|
|
@ -4801,8 +4801,10 @@ mod tests {
|
|||
if !reads {
|
||||
let error = outcome.unwrap_err();
|
||||
let typed = error.downcast_ref::<StoreUnreadableError>().unwrap_or_else(|| panic!("{what}: {error:#}"));
|
||||
let cause = std::error::Error::source(typed).unwrap_or_else(|| panic!("{what}: the cause is the source"));
|
||||
assert!(typed.to_string().contains(&cause.to_string()), "{what}: the message carries the cause: {typed}");
|
||||
let cause = std::error::Error::source(typed).unwrap_or_else(|| panic!("{what}: the cause is the source")).to_string();
|
||||
// The message is the error's own; the cause is printed once, by the chain.
|
||||
assert!(!typed.to_string().contains(&cause), "{what}: the message repeats the cause: {typed}");
|
||||
assert_eq!(format!("{error:#}").matches(&cause).count(), 1, "{what}: {error:#}");
|
||||
assert_eq!(std::fs::read(&path)?, bytes, "{what} rewrote a file it cannot read");
|
||||
} else if operation != "open" {
|
||||
assert_eq!(store_header(&std::fs::read(&path)?)?.as_deref(), Some(header), "{what}");
|
||||
|
|
@ -4821,6 +4823,22 @@ mod tests {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
// An id one entry owns and a later entry lists as compatible names the entry that owns it.
|
||||
#[test]
|
||||
fn an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = temp.path().join("store.msgpack");
|
||||
std::fs::copy(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../tests/vectors/document-variants-c2a/readability/own-id-over-compatible-v3.bin"),
|
||||
&path,
|
||||
)?;
|
||||
let envelopes = SingleFileMessagePackBackingStore::new(&path).pull_all()?;
|
||||
assert_eq!(envelopes.len(), 2);
|
||||
assert!(envelopes.iter().all(|envelope| envelope.r#type == "vectors.item"), "{envelopes:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// A store cut short is refused whatever its version, and left as it is.
|
||||
#[test]
|
||||
fn a_truncated_v1_or_v2_store_is_refused_on_open_and_rewrite() -> Result<()> {
|
||||
|
|
|
|||
|
|
@ -68,11 +68,18 @@ function inspectV1Snapshot(filePath: string, fileSizeBytes: number, decoded: [St
|
|||
}
|
||||
|
||||
const catalog = catalogRaw.map(decodeCatalogEntry);
|
||||
// An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible.
|
||||
const catalogBySchemaId = new Map<string, InspectedCatalogEntry>();
|
||||
for (const entry of catalog) {
|
||||
catalogBySchemaId.set(entry.schemaId, entry);
|
||||
if (!catalogBySchemaId.has(entry.schemaId)) {
|
||||
catalogBySchemaId.set(entry.schemaId, entry);
|
||||
}
|
||||
}
|
||||
for (const entry of catalog) {
|
||||
for (const compatibleSchemaId of entry.compatibleSchemaIds) {
|
||||
catalogBySchemaId.set(compatibleSchemaId, entry);
|
||||
if (!catalogBySchemaId.has(compatibleSchemaId)) {
|
||||
catalogBySchemaId.set(compatibleSchemaId, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
const records = recordsRaw.map((record) => decodeV1Record(record, catalogBySchemaId));
|
||||
|
|
|
|||
|
|
@ -239,9 +239,19 @@ function catalogEntriesFor(entries: CultCacheEnvelope[]): Map<string, CultCacheS
|
|||
compatibleSchemaIds: [schemaId],
|
||||
members: [],
|
||||
};
|
||||
if (!catalog.has(published.schemaId)) {
|
||||
catalog.set(published.schemaId, published);
|
||||
}
|
||||
// Entries that share an id are one schema seen by different writers: the entry written lists every id any of them lists.
|
||||
const existing = catalog.get(published.schemaId);
|
||||
catalog.set(
|
||||
published.schemaId,
|
||||
existing === undefined
|
||||
? published
|
||||
: {
|
||||
...existing,
|
||||
compatibleSchemaIds: [
|
||||
...new Set([...(existing.compatibleSchemaIds ?? [existing.schemaId]), ...(published.compatibleSchemaIds ?? [published.schemaId])]),
|
||||
],
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return catalog;
|
||||
|
|
@ -337,12 +347,19 @@ function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined {
|
|||
throw new Error("CultCache v1 snapshot must contain a schema catalog and record array.");
|
||||
}
|
||||
|
||||
// A schema id names the entry that has it as its own id; only an id no entry owns names the entry that lists it as compatible.
|
||||
const catalogEntries = catalogRaw.map(decodeCatalogEntry);
|
||||
const catalogBySchemaId = new Map<string, CultCacheSchemaCatalogEntry>();
|
||||
for (const entry of catalogRaw) {
|
||||
const catalogEntry = decodeCatalogEntry(entry);
|
||||
catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry);
|
||||
for (const catalogEntry of catalogEntries) {
|
||||
if (!catalogBySchemaId.has(catalogEntry.schemaId)) {
|
||||
catalogBySchemaId.set(catalogEntry.schemaId, catalogEntry);
|
||||
}
|
||||
}
|
||||
for (const catalogEntry of catalogEntries) {
|
||||
for (const compatibleSchemaId of catalogEntry.compatibleSchemaIds ?? []) {
|
||||
catalogBySchemaId.set(compatibleSchemaId, catalogEntry);
|
||||
if (!catalogBySchemaId.has(compatibleSchemaId)) {
|
||||
catalogBySchemaId.set(compatibleSchemaId, catalogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1344,6 +1344,61 @@ test("the store publishes a record's schema id even when the envelope's catalog
|
|||
assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 1);
|
||||
});
|
||||
|
||||
// Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists,
|
||||
// whichever order the records arrive in.
|
||||
test("the store writer merges the compatible ids of entries that share a schema id, in either order", async () => {
|
||||
const entry = (compatibleSchemaIds: string[]) => ({
|
||||
schemaId: "tests.merge.x",
|
||||
schemaName: "tests.merge",
|
||||
schemaVersion: "tests.merge.v1",
|
||||
contentHash: "tests.merge.x",
|
||||
canonicalSchemaJson: "",
|
||||
compatibleSchemaIds,
|
||||
members: [],
|
||||
});
|
||||
const record = (key: string, schemaId: string, compatibleSchemaIds: string[]) => ({
|
||||
key,
|
||||
type: "tests.merge",
|
||||
schemaId,
|
||||
payload: encode({ key }),
|
||||
storedAt: "2026-09-30T00:00:00.0000000Z",
|
||||
catalogEntry: entry(compatibleSchemaIds),
|
||||
});
|
||||
const a = record("a", "tests.merge.x", ["tests.merge.x"]);
|
||||
const b = record("b", "tests.merge.y", ["tests.merge.x", "tests.merge.y"]);
|
||||
const dir = await mkdtemp(join(tmpdir(), "cultcache-merge-"));
|
||||
|
||||
for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) {
|
||||
const file = join(dir, `${name}.msgpack`);
|
||||
await new SingleFileMessagePackBackingStore(file).pushAll([...order]);
|
||||
const decoded = decode(await readFile(file)) as [string, unknown[][], unknown[][]];
|
||||
assert.deepEqual(decoded[1].map((catalogEntry) => (catalogEntry[5] as string[]).slice().sort()), [["tests.merge.x", "tests.merge.y"]], name);
|
||||
assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2, name);
|
||||
}
|
||||
|
||||
// A put over a key of a file whose other record is stored under the wider entry's compatible id.
|
||||
const file = join(dir, "push.msgpack");
|
||||
await new SingleFileMessagePackBackingStore(file).pushAll([b]);
|
||||
await new SingleFileMessagePackBackingStore(file).push(a);
|
||||
assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2);
|
||||
});
|
||||
|
||||
test("an id an entry owns names that entry, not one that lists it as compatible", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "cultcache-own-id-"));
|
||||
const file = await copyVector(dir, join(c2aVectors, "readability", "own-id-over-compatible-v3.bin"), "own.msgpack");
|
||||
const envelopes = await new SingleFileMessagePackBackingStore(file).pullAll();
|
||||
assert.deepEqual(envelopes.map((envelope) => envelope.type), ["vectors.item", "vectors.item"]);
|
||||
const inspection = inspectCultCacheBytes("own.msgpack", await readFile(file));
|
||||
assert.deepEqual(inspection.records.map((record) => record.schemaName), ["vectors.item", "vectors.item"]);
|
||||
});
|
||||
|
||||
// The inspector reads what the reader reads: a record under an id its catalog lists only as compatible is inspected.
|
||||
test("CultCache inspector reads a record whose schema the catalog publishes only as a compatible id", async () => {
|
||||
const bytes = await readFile(join(c2aVectors, "readability", "compatible-id-only-v3.bin"));
|
||||
const inspection = inspectCultCacheBytes("compatible.msgpack", bytes);
|
||||
assert.deepEqual(inspection.records.map((record) => record.schemaName), ["vectors.item", "vectors.item"]);
|
||||
});
|
||||
|
||||
test("a record loaded under a compatible schema id is written back under the id its catalog entry carries", async () => {
|
||||
const file = join(await mkdtemp(join(tmpdir(), "cultcache-foreign-id-")), "store.msgpack");
|
||||
const payload = encode({ name: "old" });
|
||||
|
|
|
|||
|
|
@ -37,11 +37,9 @@ def apply_raw_document_record(
|
|||
) -> CultNetAppliedRecord:
|
||||
wire = record.to_wire() if isinstance(record, CultNetRawDocumentRecord) else record
|
||||
schema_id = str(wire["schemaId"])
|
||||
document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
|
||||
documents_by_schema_id,
|
||||
schema_id,
|
||||
wire,
|
||||
)
|
||||
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, wire)
|
||||
# The record is applied, stamped and reported under the local schema id, as the cache stores it and a delete reports it.
|
||||
resolved_schema_id = document.catalog_entry().schema_id
|
||||
envelope = raw_record_to_envelope(document, resolved_schema_id, wire)
|
||||
value = cache.put_envelope(document, envelope)
|
||||
return CultNetAppliedRecord(
|
||||
|
|
@ -82,11 +80,8 @@ def apply_raw_snapshot(
|
|||
if not isinstance(record, dict):
|
||||
continue
|
||||
schema_id = str(record["schemaId"])
|
||||
document, resolved_schema_id = resolve_document_and_schema_id_for_raw_record(
|
||||
documents_by_schema_id,
|
||||
schema_id,
|
||||
record,
|
||||
)
|
||||
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)
|
||||
resolved_schema_id = document.catalog_entry().schema_id
|
||||
envelope = raw_record_to_envelope(document, resolved_schema_id, record)
|
||||
batch = batches.setdefault(document.type, (document, []))
|
||||
batch[1].append((resolved_schema_id, envelope))
|
||||
|
|
|
|||
|
|
@ -1818,7 +1818,8 @@ class CultNetTests(unittest.TestCase):
|
|||
return cache
|
||||
|
||||
cache = open_cache()
|
||||
apply_raw_document_record(cache, schema_document_map([document]), record)
|
||||
applied = apply_raw_document_record(cache, schema_document_map([document]), record)
|
||||
self.assertEqual(applied.schema_id, document.catalog_entry().schema_id)
|
||||
self.assertEqual(
|
||||
cache.get_required_envelope(document, "policy:1").schema_id, document.catalog_entry().schema_id
|
||||
)
|
||||
|
|
|
|||
|
|
@ -251,11 +251,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
|||
currentIndex[key] = ToIndexRecord(stored);
|
||||
}
|
||||
|
||||
var catalogCandidates = currentManifest.SchemaCatalog
|
||||
.Concat(_durableCatalog)
|
||||
.Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()))
|
||||
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
|
||||
.Select(group => group.Last())
|
||||
var catalogCandidates = CultSchemaCatalogEntry.MergeById(
|
||||
currentManifest.SchemaCatalog
|
||||
.Concat(_durableCatalog)
|
||||
.Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())),
|
||||
preferLast: true)
|
||||
.ToArray();
|
||||
var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal);
|
||||
var targetCatalog = catalogCandidates
|
||||
|
|
@ -464,8 +464,9 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
|||
if (!string.Equals(snapshot.FormatVersion, IndexedFormatVersion, StringComparison.Ordinal) &&
|
||||
!string.Equals(snapshot.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable.");
|
||||
throw new CultStoreUnreadableException(
|
||||
$"Directory store {_manifestFile.FullName} is {snapshot.FormatVersion}; only {IndexedFormatVersion} and {IndexedFormatVersionWithIds} are readable.",
|
||||
_manifestFile.FullName);
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
|
|
|
|||
|
|
@ -28,6 +28,33 @@ namespace GameCult.Caching
|
|||
public string[] CompatibleSchemaIds { get; set; } = Array.Empty<string>();
|
||||
|
||||
public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty<CultSchemaMemberCatalogEntry>();
|
||||
|
||||
// One entry per schema id, whatever order the candidates come in: entries that share an id are one schema seen by different
|
||||
// writers, so the entry written lists every id any of them lists, and its other fields are the first's (or the last's).
|
||||
internal static IEnumerable<CultSchemaCatalogEntry> MergeById(IEnumerable<CultSchemaCatalogEntry> entries, bool preferLast = false)
|
||||
{
|
||||
foreach (var group in entries.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal))
|
||||
{
|
||||
var members = group.ToArray();
|
||||
var chosen = preferLast ? members[^1] : members[0];
|
||||
if (members.Length == 1)
|
||||
{
|
||||
yield return chosen;
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new CultSchemaCatalogEntry
|
||||
{
|
||||
SchemaId = chosen.SchemaId,
|
||||
SchemaName = chosen.SchemaName,
|
||||
SchemaVersion = chosen.SchemaVersion,
|
||||
ContentHash = chosen.ContentHash,
|
||||
CanonicalSchemaJson = chosen.CanonicalSchemaJson,
|
||||
CompatibleSchemaIds = members.SelectMany(entry => entry.CompatibleSchemaIds).Distinct(StringComparer.Ordinal).ToArray(),
|
||||
Members = chosen.Members
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CultSchemaMemberCatalogEntry
|
||||
|
|
@ -3637,9 +3664,7 @@ namespace GameCult.Caching
|
|||
var snapshot = new CultPersistedStoreSnapshot
|
||||
{
|
||||
FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)),
|
||||
SchemaCatalog = catalog
|
||||
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
|
||||
.Select(group => group.First())
|
||||
SchemaCatalog = CultSchemaCatalogEntry.MergeById(catalog)
|
||||
// An entry stays while it publishes a schema id some record carries, as its own id or a compatible one.
|
||||
.Where(entry => used.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(used.Contains))
|
||||
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
|
||||
<InternalsVisibleTo Include="GameCult.Mesh" />
|
||||
<InternalsVisibleTo Include="GameCult.Caching.MessagePack" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -127,6 +127,8 @@ namespace GameCult.Caching.Tests
|
|||
[TestCase("current-catalog-bad-canonical.msgpack")]
|
||||
[TestCase("current-catalog-bad-members.msgpack")]
|
||||
[TestCase("current-catalog-bad-compat.msgpack")]
|
||||
[TestCase("current-catalog-bad-slot5-nil.msgpack")]
|
||||
[TestCase("current-catalog-bad-slot5-int.msgpack")]
|
||||
public void SingleFileRefusesAMalformedCurrentCatalogEntry(string vector)
|
||||
{
|
||||
Assert.That(Refusal(vector), Is.Not.Empty);
|
||||
|
|
|
|||
|
|
@ -155,6 +155,67 @@ namespace GameCult.Caching.Tests
|
|||
Assert.That(rewritten.Records.Select(record => record.SchemaId), Does.Contain("vectors.old.id"));
|
||||
}
|
||||
|
||||
// An id one entry owns and a later entry lists as compatible names the entry that owns it.
|
||||
[Test]
|
||||
public void AnIdAnEntryOwnsNamesThatEntryNotOneThatListsItAsCompatible()
|
||||
{
|
||||
var path = Path.Combine(_directory, "own-id.cc");
|
||||
File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "own-id-over-compatible-v3.bin")));
|
||||
|
||||
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
|
||||
Assert.That(cache.Get<VectorItem>(new CultRecordKey("alpha")), Is.Not.Null, "the record read as the schema that owns its id");
|
||||
}
|
||||
|
||||
// A directory rewrite where the entry that lists a kept record's id shares its own id with the schema being written keeps that
|
||||
// id in the entry it writes, so the manifest still reads.
|
||||
[Test]
|
||||
public void ADirectoryRewriteMergesEntriesThatShareAnIdAndKeepsEveryIdTheyList()
|
||||
{
|
||||
var path = DirectoryStore("merged.cc");
|
||||
var manifest = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||
var entry = manifest.SchemaCatalog.Single();
|
||||
manifest.Records.Single().SchemaId = "vectors.old.id";
|
||||
entry.CompatibleSchemaIds = new[] { entry.SchemaId, "vectors.old.id" };
|
||||
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(manifest));
|
||||
|
||||
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }))
|
||||
cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e")));
|
||||
|
||||
var rewritten = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||
Assert.That(rewritten.SchemaCatalog.Single().CompatibleSchemaIds, Does.Contain("vectors.old.id"));
|
||||
}
|
||||
|
||||
// Entries that share an id are merged whatever order they arrive in: every id any of them lists survives.
|
||||
[Test]
|
||||
public void EntriesThatShareAnIdMergeTheirCompatibleIdsInEitherOrder()
|
||||
{
|
||||
CultSchemaCatalogEntry Entry(params string[] compatible) => new()
|
||||
{
|
||||
SchemaId = "x", SchemaName = "n", SchemaVersion = "n.v1", ContentHash = "h", CompatibleSchemaIds = compatible
|
||||
};
|
||||
var plain = Entry("x");
|
||||
var wide = Entry("x", "y");
|
||||
|
||||
foreach (var order in new[] { new[] { plain, wide }, new[] { wide, plain } })
|
||||
foreach (var preferLast in new[] { false, true })
|
||||
{
|
||||
var merged = CultSchemaCatalogEntry.MergeById(order, preferLast).Single();
|
||||
Assert.That(merged.CompatibleSchemaIds, Is.EquivalentTo(new[] { "x", "y" }));
|
||||
}
|
||||
}
|
||||
|
||||
// A directory manifest in a format the directory store does not read is refused with the typed exception, naming it.
|
||||
[Test]
|
||||
public void ADirectoryManifestInAnotherFormatIsRefusedNamingIt()
|
||||
{
|
||||
var path = Path.Combine(_directory, "wrong-format.cc");
|
||||
File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "..", "v3-base.msgpack")));
|
||||
|
||||
Assert.That(
|
||||
() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true }),
|
||||
Throws.TypeOf<CultStoreUnreadableException>().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path));
|
||||
}
|
||||
|
||||
// A commit that lands onto the file as it is keeps the header the file carries: a store already v3 stays v3.
|
||||
[TestCase("../v3-base.msgpack", "cultcache.store.v3")]
|
||||
[TestCase("compatible-id-only-v3.bin", "cultcache.store.v3")]
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ public sealed class CultMeshVariantRefusalTests
|
|||
[TestCase("current-catalog-bad-canonical.msgpack")]
|
||||
[TestCase("current-catalog-bad-members.msgpack")]
|
||||
[TestCase("current-catalog-bad-compat.msgpack")]
|
||||
[TestCase("current-catalog-bad-slot5-nil.msgpack")]
|
||||
[TestCase("current-catalog-bad-slot5-int.msgpack")]
|
||||
public void ReadSingleFileDocument_RefusesAMalformedCurrentCatalogEntry(string vector)
|
||||
{
|
||||
Refusal(vector).Path.Should().Be(VectorPath(vector));
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -18,6 +18,10 @@ at test time. Run: python generate.py (needs msgpack).
|
|||
v1 header, plain records, and the first catalog entry of the current layout with one slot
|
||||
malformed (canonical schema json an int; members an int; a compatible id an int). Not the
|
||||
older layout, so CultMesh refuses them as CultCache does
|
||||
current-catalog-bad-slot5-nil.msgpack
|
||||
current-catalog-bad-slot5-int.msgpack
|
||||
the same, with slot 5 (the compatible ids) nil or an integer: neither is a string, so neither is
|
||||
the older layout
|
||||
v1-base.msgpack NOT generated here: written once by the Python runtime at CultLib
|
||||
69a21bb (SingleFileMessagePackBackingStore.push of alpha and
|
||||
beta, type vectors.item, before any C0 code existed)
|
||||
|
|
@ -60,6 +64,8 @@ for name, entry in (
|
|||
("current-catalog-bad-canonical", current[:4] + [5]),
|
||||
("current-catalog-bad-members", current[:6] + [5]),
|
||||
("current-catalog-bad-compat", current[:5] + [[7]] + current[6:]),
|
||||
("current-catalog-bad-slot5-nil", current[:5] + [None] + current[6:]),
|
||||
("current-catalog-bad-slot5-int", current[:5] + [7] + current[6:]),
|
||||
):
|
||||
# One catalog entry and the records that use it: the older-layout reader takes an entry's slot 5 for a string, so a second
|
||||
# current-layout entry would make it refuse for that reason, not for the malformed slot under test.
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ vectors = {
|
|||
"missing-schema-versioned-v3": pack([base[0], [], [record[:3] + [pack(["vectors.item.v1", "n", 1])] for record in base[2]]]),
|
||||
# The record names its schema by an id the catalog publishes only as a compatible id of an entry with another id.
|
||||
"compatible-id-only-v3": pack([base[0], [[base[1][0][0] + ".next"] + base[1][0][1:]], base[2]]),
|
||||
# An id that one entry owns and a later entry lists as compatible names the entry that owns it (here, vectors.item).
|
||||
"own-id-over-compatible-v3": pack([base[0], base[1] + [["vectors.item.next", "vectors.other", "vectors.other.v1", "vectors.item.next", "{}", ["vectors.item"], []]], base[2]]),
|
||||
# v3-base plus a variant of alpha: a fifth record slot, [baseKey, overrides[]], over an empty payload, overriding the name. Only C# reads it.
|
||||
"variant-slot-v3": pack([base[0], base[1], base[2] + [["gamma"] + base[2][0][1:3] + [b"", ["alpha", [[0, [[0, ""]], "", "gamma"]]]]]]),
|
||||
# v3-base with a fourth top-level slot after the records.
|
||||
|
|
|
|||
|
|
@ -24,3 +24,4 @@ missing-schema-v3.bin refuses refuses refuses refuses
|
|||
variant-slot-v3.bin reads refuses refuses refuses
|
||||
missing-schema-versioned-v3.bin refuses refuses refuses refuses
|
||||
compatible-id-only-v3.bin reads reads reads reads
|
||||
own-id-over-compatible-v3.bin reads reads reads reads
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
“²cultcache.store.v3’—¬vectors.item¬vectors.item¯vectors.item.v1¬vectors.itemÙL{"schemaName":"vectors.item","schemaVersion":"vectors.item.v1","members":[]}‘¬vectors.item<65>—±vectors.item.nextvectors.other°vectors.other.v1±vectors.item.next¢{}‘¬vectors.item<65>’”¥alpha¬vectors.item¹2026-09-29T00:00:00+00:00Ä’¥alpha”¤beta¬vectors.item¹2026-09-29T00:00:01+00:00Ä’¤beta
|
||||
Loading…
Reference in a new issue