From e77246b657f90007f7441abca2fc8513e864def2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 10:03:34 +0200 Subject: [PATCH] Writers derive the catalog from the records they write; the merging is deleted Every writer (C# single-file and directory, TypeScript, Python; Rust already did) chooses, for each schema id a record carries, one entry that publishes it: the entry that owns the id (a registered descriptor, else an arrived entry), else one that lists it as compatible. A registered descriptor wins over an arrived entry with the same own id. Entries are written as chosen, with no union of compatible ids and no dependence on arrival order; an entry no record needs is not written. Two entries of one tier that share an own id and disagree on the schema name, records of different types under one id, or a record no chosen entry publishes refuse the write with a typed error (CultSchemaConflictException, SchemaConflictError) naming the id, the names and a record key, and the store is left as it was. CultSchemaCatalogEntry.MergeById and the Python and TypeScript merge paths are gone. A cache-held record unchanged since the file was read is rewritten under its registered id (C# single-file and directory), so an entry that lists an older id is not needed to publish it. Changelogs for cultnet-py (removed export) and cultcache-py. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultcache-py/CHANGELOG.md | 8 + .../cultcache-py/src/cultcache_py/__init__.py | 3 +- .../cultcache-py/src/cultcache_py/stores.py | 132 +++++++----- packages/cultcache-py/tests/test_cultcache.py | 160 ++++++++++----- packages/cultcache-rs/src/lib.rs | 60 +++++- packages/cultcache-ts/src/index.ts | 2 +- .../single-file-messagepack-backing-store.ts | 190 +++++++++++------- packages/cultcache-ts/src/store-format.ts | 19 ++ packages/cultcache-ts/test/cult-cache.test.ts | 182 +++++++++++------ packages/cultnet-py/CHANGELOG.md | 9 + .../DirectoryMessagePackBackingStore.cs | 26 ++- .../Contracts/cultcache-persistence-format.md | 17 +- src/GameCult.Caching/CultCache.cs | 133 +++++++++--- .../CultSchemaConflictException.cs | 24 +++ 14 files changed, 687 insertions(+), 278 deletions(-) create mode 100644 packages/cultcache-py/CHANGELOG.md create mode 100644 packages/cultnet-py/CHANGELOG.md create mode 100644 src/GameCult.Caching/CultSchemaConflictException.cs diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md new file mode 100644 index 00000000..70a59b76 --- /dev/null +++ b/packages/cultcache-py/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## Unreleased + +- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a + write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload. +- The single-file writer derives the catalog from the records it writes: one entry per carried schema id, the registered entry over an + arrived one, written as chosen and never merged. A cache stamps a record with its registered schema's id. diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index 84aaacf5..f0ae38b2 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -9,7 +9,7 @@ from .documents import ( define_document_registry, define_document_type, ) -from .stores import JsonLinesBackingStore, SingleFileMessagePackBackingStore, StoreUnreadableError +from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError __all__ = [ "BackingStore", @@ -21,6 +21,7 @@ __all__ = [ "DatabaseEntryField", "DocumentDefinition", "JsonLinesBackingStore", + "SchemaConflictError", "SingleFileMessagePackBackingStore", "StoreUnreadableError", "database_entry_field", diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index f06104b1..78de24f8 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -4,7 +4,6 @@ import base64 import json import os import threading -from dataclasses import replace from pathlib import Path from typing import Any @@ -34,6 +33,20 @@ class StoreUnreadableError(ValueError): self.path = path +class SchemaConflictError(ValueError): + """A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id + and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names + involved (`schema_names`) and a record key (`record_key`).""" + + def __init__(self, schema_id: str, schema_names: list[str], record_key: str) -> None: + super().__init__( + f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}" + ) + self.schema_id = schema_id + self.schema_names = schema_names + self.record_key = record_key + + class JsonLinesBackingStore: def __init__(self, path: str | os.PathLike[str]) -> None: self.path = Path(path) @@ -125,27 +138,21 @@ class SingleFileMessagePackBackingStore: def push(self, envelope: CultCacheEnvelope) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - existing[(envelope.type, envelope.key)] = envelope - self._replace_all(list(existing.values())) + self._replace_all([envelope], self.pull_all()) def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - for envelope in envelopes: - existing[(envelope.type, envelope.key)] = envelope - self._replace_all(list(existing.values())) + self._replace_all(list(envelopes), self.pull_all()) def delete(self, type: str, key: str) -> None: with self._lock: - existing = [item for item in self.pull_all() if not (item.type == type and item.key == key)] - self._replace_all(existing) + self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)]) - def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: msgpack = self._msgpack() self.path.parent.mkdir(parents=True, exist_ok=True) temp = self.path.with_suffix(self.path.suffix + ".tmp") - temp.write_bytes(msgpack.packb(_encode_snapshot(envelopes, self._format), use_bin_type=True)) + temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) temp.replace(self.path) @staticmethod @@ -160,46 +167,81 @@ class SingleFileMessagePackBackingStore: return msgpack -def _encode_snapshot(envelopes: list[CultCacheEnvelope], format_version: str) -> list[Any]: - # Every record's schema id is published by a catalog entry: the entry the envelope carries when it publishes that id (as its - # own id or a compatible one), else a default entry under the record's id. The catalog is keyed by the entry's own id, so a - # record read under a compatible id is written back beside the entry that lists it. - catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {} +def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]: + """Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in.""" + return sorted(candidates, key=lambda entry: (entry.schema_name, entry.content_hash, ",".join(entry.compatible_schema_ids))) - def place(envelope: CultCacheEnvelope, candidates: list[CultCacheSchemaCatalogEntry]) -> None: - for candidate in candidates: - existing = catalog_by_schema_id.get(candidate.schema_id) - if existing is None: - catalog_by_schema_id[candidate.schema_id] = candidate - return - # Entries that share an id and a schema name are one schema seen by different writers: the entry written lists every id - # any of them lists. An entry of another schema is never merged: the record gets a default entry under its own id. - if existing.schema_name == candidate.schema_name: - catalog_by_schema_id[candidate.schema_id] = replace( - existing, - compatible_schema_ids=tuple(dict.fromkeys((*existing.compatible_schema_ids, *candidate.compatible_schema_ids))), - ) - return - raise ValueError(f"CultCache schema id {_schema_id_for(envelope)!r} cannot identify two different schemas") - # A record's own id is claimed first, so an entry that lists it as compatible cannot take it from the schema that owns it, - # whatever order the records arrive in. - listed: list[tuple[CultCacheEnvelope, CultCacheSchemaCatalogEntry]] = [] - for envelope in envelopes: +def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultCacheSchemaCatalogEntry]: + """The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, + one entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one + that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an + arrived one with the same own id. Records of different types under one id, two arrived entries with the same own id and + different schema names, or a record no chosen entry publishes, refuse the write. `records` are (envelope, registered): an + envelope the caller supplied is registered, one read back from the file is arrived.""" + entries: list[tuple[CultCacheSchemaCatalogEntry, bool]] = [] + by_id: dict[str, list[CultCacheEnvelope]] = {} + for envelope, registered in records: schema_id = _schema_id_for(envelope) supplied = envelope.catalog_entry - if supplied is not None and supplied.schema_id != schema_id and schema_id in supplied.compatible_schema_ids: - listed.append((envelope, supplied)) - elif supplied is not None and supplied.schema_id == schema_id: - place(envelope, [supplied]) - else: - place(envelope, [_default_catalog_entry(envelope)]) - for envelope, supplied in listed: - place(envelope, [supplied, _default_catalog_entry(envelope)]) + publishes = supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids) + entries.append((supplied if publishes else _default_catalog_entry(envelope), registered)) + by_id.setdefault(schema_id, []).append(envelope) + chosen: dict[str, tuple[CultCacheSchemaCatalogEntry, bool]] = {} + for schema_id in sorted(by_id): + carrying = by_id[schema_id] + record_key = min(envelope.key for envelope in carrying) + types = sorted({envelope.type for envelope in carrying}) + if len(types) > 1: + raise SchemaConflictError(schema_id, types, record_key) + own_registered = next(iter(canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)), None) + own_arrived = canonical(entry for entry, registered in entries if not registered and entry.schema_id == schema_id) + own_registered_all = canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id) + # Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + names = sorted({entry.schema_name for entry in (own_registered_all or own_arrived)}) + if len(names) > 1: + raise SchemaConflictError(schema_id, names, record_key) + if own_registered is not None: + pick, registered_pick = own_registered, True + elif own_arrived: + pick, registered_pick = own_arrived[0], False + else: + listed_registered = next(iter(canonical(entry for entry, registered in entries if registered and schema_id in entry.compatible_schema_ids)), None) + listed_arrived = next(iter(canonical(entry for entry, registered in entries if not registered and schema_id in entry.compatible_schema_ids)), None) + pick = listed_registered if listed_registered is not None else listed_arrived + registered_pick = listed_registered is not None + if pick is None: + continue + prior = chosen.get(pick.schema_id) + if prior is not None: + if prior[1] and not registered_pick: + continue + # Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other. + if prior[1] == registered_pick and prior[0].schema_name != pick.schema_name: + raise SchemaConflictError(pick.schema_id, sorted({prior[0].schema_name, pick.schema_name}), record_key) + chosen[pick.schema_id] = (pick, registered_pick) + + for schema_id in sorted(by_id): + if not any(entry.schema_id == schema_id or schema_id in entry.compatible_schema_ids for entry, _ in chosen.values()): + raise SchemaConflictError( + schema_id, sorted(entry.schema_name for entry, _ in chosen.values()), min(envelope.key for envelope in by_id[schema_id]) + ) + return [entry for entry, _ in chosen.values()] + + +def _encode_snapshot( + supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str +) -> list[Any]: + replaced = {(envelope.type, envelope.key) for envelope in supplied} + kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced] + envelopes = [*kept, *supplied] catalog = [ _encode_catalog_entry(entry) - for entry in sorted(catalog_by_schema_id.values(), key=lambda item: item.schema_name) + for entry in sorted( + _derive_catalog([*((envelope, False) for envelope in kept), *((envelope, True) for envelope in supplied)]), + key=lambda item: (item.schema_name, item.schema_id), + ) ] records = [ [envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload] diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index ffbc2706..408afd6c 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -315,68 +315,130 @@ class CultCacheTests(unittest.TestCase): self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"]) self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1) - # Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists, - # whichever order the records arrive in. - def test_the_store_writer_merges_the_compatible_ids_of_entries_that_share_a_schema_id_in_either_order(self) -> None: - import msgpack # type: ignore + # The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever + # order the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id. + @staticmethod + def _writer_entry(schema_id: str, name: str, content_hash: str, compatible: tuple[str, ...]): from cultcache_py import CultCacheSchemaCatalogEntry - from cultcache_py.stores import _encode_snapshot - def entry(compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry: - return CultCacheSchemaCatalogEntry( - schema_id="tests.merge.x", schema_name="tests.merge", schema_version="tests.merge.v1", - content_hash="tests.merge.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(), - ) + return CultCacheSchemaCatalogEntry( + schema_id=schema_id, schema_name=name, schema_version=name + ".v1", content_hash=content_hash, + canonical_schema_json="", compatible_schema_ids=compatible, members=(), + ) - def record(key: str, schema_id: str, compatible: tuple[str, ...]) -> CultCacheEnvelope: - return CultCacheEnvelope( - key=key, type="tests.merge", payload=msgpack.packb({"key": key}, use_bin_type=True), - stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=entry(compatible), - ) + def _writer_record(self, key: str, type: str, schema_id: str, catalog_entry=None) -> CultCacheEnvelope: + import msgpack # type: ignore - a = record("a", "tests.merge.x", ("tests.merge.x",)) - b = record("b", "tests.merge.y", ("tests.merge.x", "tests.merge.y")) - for order in ([a, b], [b, a]): - catalog = _encode_snapshot(order, "cultcache.store.v1")[1] - self.assertEqual([sorted(entry[5]) for entry in catalog], [["tests.merge.x", "tests.merge.y"]]) + return CultCacheEnvelope( + key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True), + stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry, + ) + + @staticmethod + def _raw_entry(entry) -> list: + return [entry.schema_id, entry.schema_name, entry.schema_version, entry.content_hash, "", list(entry.compatible_schema_ids), []] + + def test_a_registered_entry_is_written_as_it_is_over_the_arrived_entries_with_the_same_own_id_in_either_order(self) -> None: + import msgpack # type: ignore + + stale1 = self._writer_entry("tests.x", "tests.n", "stale-1", ("tests.x", "old")) + stale2 = self._writer_entry("tests.x", "tests.n", "stale-2", ("tests.x",)) + registered = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "fresh", ("tests.x",))) + for entries in ([stale1, stale2], [stale2, stale1]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(entry) for entry in entries], + [["b", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"b": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push(registered) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual(len(catalog), 1) + self.assertEqual(catalog[0][3], "fresh") + self.assertEqual(catalog[0][5], ["tests.x"], "written as chosen, no union with the arrived entries' ids") + + def test_a_rename_with_a_stable_schema_id_a_whole_view_write_names_the_registered_schema(self) -> None: + import msgpack # type: ignore with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "store.msgpack" - SingleFileMessagePackBackingStore(path).push_all([b]) - SingleFileMessagePackBackingStore(path).push(a) - self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2) + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.old", "stale", ("tests.x",)))], + [["a", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"a": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push_all( + [self._writer_record("a", "tests.new", "tests.x", self._writer_entry("tests.x", "tests.new", "fresh", ("tests.x",)))] + ) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual([(entry[1], entry[3]) for entry in catalog], [("tests.new", "fresh")]) + + def test_two_entries_of_one_tier_that_share_an_own_id_and_disagree_on_the_schema_name_refuse_the_write_typed(self) -> None: + from cultcache_py import SchemaConflictError + + first = self._writer_record("b", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.first", "h1", ("tests.x",))) + second = self._writer_record("a", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.second", "h2", ("tests.x",))) + for order in ([first, second], [second, first]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.x") + self.assertEqual(refused.exception.record_key, "a") + self.assertEqual(refused.exception.schema_names, ["tests.first", "tests.second"]) + self.assertFalse(path.exists()) + + def test_records_of_different_types_under_one_schema_id_refuse_the_write_typed_and_no_record_changes_type(self) -> None: + from cultcache_py import SchemaConflictError + + a = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y"))) + b = self._writer_record("b", "tests.m", "tests.y", self._writer_entry("tests.y", "tests.m", "h2", ("tests.y",))) + for order in ([a, b], [b, a]): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.y") + self.assertEqual(refused.exception.schema_names, ["tests.m", "tests.n"]) + + def test_an_entry_that_owns_an_id_is_chosen_over_one_that_lists_it(self) -> None: + import msgpack # type: ignore + + owner = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.y", "tests.n", "h1", ("tests.y",))) + lister = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.y"))) + for order in ([lister, owner], [owner, lister]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all(order) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual(sorted((entry[0], entry[5]) for entry in catalog), [("tests.x", ["tests.x", "tests.y"]), ("tests.y", ["tests.y"])]) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all([owner]) + self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.y"]) + + def test_a_record_no_chosen_entry_publishes_refuses_the_write_typed_and_leaves_the_file(self) -> None: + import msgpack # type: ignore + from cultcache_py import SchemaConflictError + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.legacy", "stale", ("tests.x", "old")))], + [["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + before = path.read_bytes() + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push( + self._writer_record("a", "tests.legacy", "tests.x", self._writer_entry("tests.x", "tests.legacy", "fresh", ("tests.x",))) + ) + self.assertEqual(refused.exception.schema_id, "old") + self.assertEqual(refused.exception.record_key, "z") + self.assertEqual(path.read_bytes(), before) def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None: for vector in ("own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"): store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / vector) self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"], vector) - # Entries of different schemas that share an id are not merged: the record gets a default entry under its own id, so no record - # is read as another schema, whichever order the records arrive in. - def test_the_store_writer_never_merges_an_entry_of_another_schema(self) -> None: - import msgpack # type: ignore - from cultcache_py import CultCacheSchemaCatalogEntry - from cultcache_py.stores import _decode_snapshot, _encode_snapshot - - def entry(name: str, compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry: - return CultCacheSchemaCatalogEntry( - schema_id="tests.retype.x", schema_name=name, schema_version=name + ".v1", - content_hash="tests.retype.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(), - ) - - def record(key: str, type: str, schema_id: str, catalog_entry: CultCacheSchemaCatalogEntry) -> CultCacheEnvelope: - return CultCacheEnvelope( - key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True), - stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry, - ) - - a = record("a", "tests.retype.mine", "tests.retype.x", entry("tests.retype.mine", ("tests.retype.x",))) - b = record("b", "tests.retype.theirs", "tests.retype.y", entry("tests.retype.theirs", ("tests.retype.x", "tests.retype.y"))) - for order in ([a, b], [b, a]): - decoded = msgpack.unpackb(msgpack.packb(_encode_snapshot(order, "cultcache.store.v1"), use_bin_type=True), raw=False) - _, envelopes = _decode_snapshot(decoded) - self.assertEqual(sorted((e.key, e.type) for e in envelopes), [("a", "tests.retype.mine"), ("b", "tests.retype.theirs")]) - def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None: import msgpack # type: ignore diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 0be95fed..fbc905cf 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -389,6 +389,27 @@ impl std::error::Error for StoreUnreadableError { } } +/// A write the catalog cannot describe: records of different types under one schema id. This writer derives one entry per carried +/// schema id from its records, so that is its only conflict. Nothing is written. It names the id, the two names and a record key. +#[derive(Debug)] +pub struct SchemaConflictError { + pub schema_id: String, + pub schema_names: Vec, + pub record_key: String, +} + +impl std::fmt::Display for SchemaConflictError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "schema id {:?} cannot describe record {:?}: {:?}", + self.schema_id, self.record_key, self.schema_names + ) + } +} + +impl std::error::Error for SchemaConflictError {} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PushAllOptions { pub soft: bool, @@ -2592,11 +2613,13 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope], format: &str) -> Result< .clone() .unwrap_or_else(|| entry.r#type.clone()); if let Some(existing_type) = schema_types.insert(schema_id.clone(), entry.r#type.clone()) { - ensure!( - existing_type == entry.r#type, - "CultCache schema {schema_id:?} cannot identify both {existing_type:?} and {:?}", - entry.r#type - ); + if existing_type != entry.r#type { + return Err(anyhow::Error::new(SchemaConflictError { + schema_id, + schema_names: vec![existing_type, entry.r#type.clone()], + record_key: entry.key.clone(), + })); + } } } @@ -4842,6 +4865,33 @@ mod tests { Ok(()) } + // Records of different types under one schema id cannot be described by one catalog entry: the write is refused, typed, in + // either order, and the file is left as it was. A rename with a stable id (same type name in the new schema) writes fine. + #[test] + fn records_of_different_types_under_one_schema_id_refuse_the_write_typed() -> Result<()> { + let record = |key: &str, r#type: &str| CultCacheEnvelope { + key: key.into(), + r#type: r#type.into(), + payload: b"one".to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some("tests.x".into()), + }; + for order in [vec![record("a", "tests.n"), record("b", "tests.m")], vec![record("b", "tests.m"), record("a", "tests.n")]] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + let error = store.push_all(&order, PushAllOptions::default()).unwrap_err(); + let conflict = error.downcast_ref::().unwrap_or_else(|| panic!("{error:#}")); + assert_eq!(conflict.schema_id, "tests.x"); + let mut names = conflict.schema_names.clone(); + names.sort(); + assert_eq!(names, vec!["tests.m".to_string(), "tests.n".to_string()]); + assert!(conflict.record_key == "a" || conflict.record_key == "b"); + assert!(!path.exists(), "nothing is written"); + } + Ok(()) + } + // A store cut short is refused whatever its version, and left as it is. #[test] fn a_truncated_v1_or_v2_store_is_refused_on_open_and_rewrite() -> Result<()> { diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 4f1388e1..a2f45d53 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,7 @@ export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; export { defineDocumentRegistry, defineDocumentType } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; -export { StoreUnreadableError } from "./store-format"; +export { SchemaConflictError, StoreUnreadableError } from "./store-format"; export type { AnyCultCacheDocumentDefinition, CacheBackingStore, diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index b8227ba2..23fa9ab4 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -4,7 +4,7 @@ import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; -import { STORE_FORMAT_VERSION, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format"; +import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, CultCacheEnvelope, @@ -49,7 +49,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const disk = await this.#readDisk(); this.#format = disk.format; if (disk.repairedLegacyPayload) { - await this.#writeAll(disk.envelopes); + await this.#writeAll([], disk.envelopes); } return disk.envelopes; @@ -61,8 +61,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const filtered = existing.filter( (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), ); - filtered.push(entry); - await this.#writeAll(filtered); + await this.#writeAll([entry], filtered); }); } @@ -72,7 +71,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const filtered = existing.filter( (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), ); - await this.#writeAll(filtered); + await this.#writeAll([], filtered); }); } @@ -95,7 +94,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { // reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused // and left as it is. this.#format = (await this.#readDisk()).format; - await this.#writeAll(entries); + await this.#writeAll(entries, []); }); } @@ -140,7 +139,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { return result; } - async #writeAll(entries: CultCacheEnvelope[]): Promise { + async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise { await mkdir(dirname(this.filePath), { recursive: true }); const tempPath = `${this.filePath}.tmp-${process.pid}-${Date.now()}-${Math.random() @@ -148,7 +147,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { .slice(2)}`; try { - await writeFile(tempPath, encode(encodeSnapshot(entries, this.#format))); + await writeFile(tempPath, encode(encodeSnapshot(supplied, arrived, this.#format))); await renameWithRetry(tempPath, this.filePath); } catch (error) { await rm(tempPath, { force: true }).catch(() => undefined); @@ -196,9 +195,14 @@ type DecodedSnapshot = { records: PersistedRecord[]; }; -function encodeSnapshot(entries: CultCacheEnvelope[], format: StoreFormat): unknown[] { - const catalog = [...catalogEntriesFor(entries).values()] - .sort((left, right) => compareOrdinal(left.schemaName, right.schemaName)); +function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] { + const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`)); + const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`)); + const entries = [...kept, ...supplied]; + const catalog = catalogEntriesFor([ + ...kept.map((envelope) => ({ envelope, registered: false })), + ...supplied.map((envelope) => ({ envelope, registered: true })), + ]).sort((left, right) => compareOrdinal(left.schemaName, right.schemaName) || compareOrdinal(left.schemaId, right.schemaId)); const records = [...entries] .sort((left, right) => compareOrdinal(left.key, right.key)) .map((entry) => [ @@ -215,71 +219,113 @@ function encodeSnapshot(entries: CultCacheEnvelope[], format: StoreFormat): unkn ]; } -// Every record's schema id is published by a catalog entry: the entry the envelope carries when it publishes that id (as its -// own id or a compatible one), else a default entry under the record's id. The catalog is keyed by the entry's own id, so a -// record read under a compatible id is written back beside the entry that lists it. -function catalogEntriesFor(entries: CultCacheEnvelope[]): Map { - const catalog = new Map(); - const defaultEntryFor = (entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry => { - const schemaId = schemaIdForEnvelope(entry); - return { - schemaId, +// The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one +// entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one that +// lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an arrived +// one with the same own id. Records of different types under one id, two arrived entries with the same own id and different schema +// names, or a record no chosen entry publishes, refuse the write. An envelope the caller supplied is registered; one read back +// from the file is arrived. +function catalogEntriesFor( + records: Array<{ envelope: CultCacheEnvelope; registered: boolean }>, +): CultCacheSchemaCatalogEntry[] { + const entries: Array<{ entry: CultCacheSchemaCatalogEntry; registered: boolean }> = []; + const byId = new Map(); + for (const { envelope, registered } of records) { + const schemaId = schemaIdForEnvelope(envelope); + const supplied = envelope.catalogEntry; + const publishes = + supplied !== undefined && (supplied.schemaId === schemaId || (supplied.compatibleSchemaIds ?? []).includes(schemaId)); + entries.push({ entry: publishes && supplied ? supplied : defaultEntryFor(envelope), registered }); + byId.set(schemaId, [...(byId.get(schemaId) ?? []), envelope]); + } + + const lists = (entry: CultCacheSchemaCatalogEntry, schemaId: string): boolean => + (entry.compatibleSchemaIds ?? [entry.schemaId]).includes(schemaId); + const chosen = new Map(); + for (const schemaId of [...byId.keys()].sort(compareOrdinal)) { + const carrying = byId.get(schemaId)!; + const recordKey = carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!; + const types = [...new Set(carrying.map((envelope) => envelope.type))].sort(compareOrdinal); + if (types.length > 1) { + throw new SchemaConflictError(schemaId, types, recordKey); + } + + const ownRegistered = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId))[0]; + const ownArrived = canonical(entries.filter((candidate) => !candidate.registered && candidate.entry.schemaId === schemaId)); + // Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + const ownRegisteredAll = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId)); + const names = [ + ...new Set((ownRegisteredAll.length > 0 ? ownRegisteredAll : ownArrived).map((candidate) => candidate.entry.schemaName)), + ].sort(compareOrdinal); + if (names.length > 1) { + throw new SchemaConflictError(schemaId, names, recordKey); + } + + let pick: { entry: CultCacheSchemaCatalogEntry; registered: boolean } | undefined; + if (ownRegistered !== undefined) { + pick = ownRegistered; + } else if (ownArrived.length > 0) { + pick = ownArrived[0]; + } else { + pick = + canonical(entries.filter((candidate) => candidate.registered && lists(candidate.entry, schemaId)))[0] ?? + canonical(entries.filter((candidate) => !candidate.registered && lists(candidate.entry, schemaId)))[0]; + } + + if (pick === undefined) { + continue; + } + + const prior = chosen.get(pick.entry.schemaId); + if (prior !== undefined) { + if (prior.registered && !pick.registered) { + continue; + } + + // Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other. + if (prior.registered === pick.registered && prior.entry.schemaName !== pick.entry.schemaName) { + throw new SchemaConflictError(pick.entry.schemaId, [prior.entry.schemaName, pick.entry.schemaName].sort(compareOrdinal), recordKey); + } + } + + chosen.set(pick.entry.schemaId, pick); + } + + for (const [schemaId, carrying] of byId) { + if (![...chosen.values()].some(({ entry }) => entry.schemaId === schemaId || lists(entry, schemaId))) { + throw new SchemaConflictError( + schemaId, + [...chosen.values()].map(({ entry }) => entry.schemaName).sort(compareOrdinal), + carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!, + ); + } + } + + return [...chosen.values()].map(({ entry }) => entry); +} + +// Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in. +function canonical(candidates: T[]): T[] { + const key = (candidate: T): string => + [candidate.entry.schemaName, candidate.entry.contentHash ?? "", (candidate.entry.compatibleSchemaIds ?? []).join(",")].join("\u0000"); + return [...candidates].sort((left, right) => compareOrdinal(key(left), key(right))); +} + +function defaultEntryFor(entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry { + const schemaId = schemaIdForEnvelope(entry); + return { + schemaId, + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + contentHash: schemaId, + canonicalSchemaJson: JSON.stringify({ schemaName: entry.type, schemaVersion: `${entry.type}.v1`, - contentHash: schemaId, - canonicalSchemaJson: JSON.stringify({ - schemaName: entry.type, - schemaVersion: `${entry.type}.v1`, - members: [], - }), - compatibleSchemaIds: [schemaId], members: [], - }; + }), + compatibleSchemaIds: [schemaId], + members: [], }; - const place = (entry: CultCacheEnvelope, candidates: CultCacheSchemaCatalogEntry[]): void => { - for (const candidate of candidates) { - const existing = catalog.get(candidate.schemaId); - if (existing === undefined) { - catalog.set(candidate.schemaId, candidate); - return; - } - - // Entries that share an id and a schema name are one schema seen by different writers: the entry written lists every id any - // of them lists. An entry of another schema is never merged: the record gets a default entry under its own id. - if (existing.schemaName === candidate.schemaName) { - catalog.set(candidate.schemaId, { - ...existing, - compatibleSchemaIds: [ - ...new Set([...(existing.compatibleSchemaIds ?? [existing.schemaId]), ...(candidate.compatibleSchemaIds ?? [candidate.schemaId])]), - ], - }); - return; - } - } - - throw new Error(`CultCache schema id "${schemaIdForEnvelope(entry)}" cannot identify two different schemas.`); - }; - - // A record's own id is claimed first, so an entry that lists it as compatible cannot take it from the schema that owns it, - // whatever order the records arrive in. - const listed: Array<[CultCacheEnvelope, CultCacheSchemaCatalogEntry]> = []; - for (const entry of entries) { - const schemaId = schemaIdForEnvelope(entry); - const supplied = entry.catalogEntry; - if (supplied !== undefined && supplied.schemaId !== schemaId && (supplied.compatibleSchemaIds ?? []).includes(schemaId)) { - listed.push([entry, supplied]); - } else if (supplied !== undefined && supplied.schemaId === schemaId) { - place(entry, [supplied]); - } else { - place(entry, [defaultEntryFor(entry)]); - } - } - - for (const [entry, supplied] of listed) { - place(entry, [supplied, defaultEntryFor(entry)]); - } - - return catalog; } function schemaIdForEnvelope(entry: CultCacheEnvelope): string { diff --git a/packages/cultcache-ts/src/store-format.ts b/packages/cultcache-ts/src/store-format.ts index 9c0a6fa5..37d12761 100644 --- a/packages/cultcache-ts/src/store-format.ts +++ b/packages/cultcache-ts/src/store-format.ts @@ -32,6 +32,25 @@ export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...u return true; } +/** + * A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id and + * disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names involved and + * a record key. + */ +export class SchemaConflictError extends Error { + readonly schemaId: string; + readonly schemaNames: string[]; + readonly recordKey: string; + + constructor(schemaId: string, schemaNames: string[], recordKey: string) { + super(`Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`); + this.name = "SchemaConflictError"; + this.schemaId = schemaId; + this.schemaNames = schemaNames; + this.recordKey = recordKey; + } +} + /** * A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra slot), * a header or record it does not know, or a body it cannot decode. Open, flush and every rewrite refuse a file with this diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 21b37e1b..76836ead 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -15,7 +15,7 @@ import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; import { defineDocumentRegistry, defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; -import { StoreUnreadableError } from "../src/store-format"; +import { SchemaConflictError, StoreUnreadableError } from "../src/store-format"; import type { CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; const execFileAsync = promisify(execFile); @@ -1344,78 +1344,138 @@ test("the store publishes a record's schema id even when the envelope's catalog assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 1); }); -// Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists, -// whichever order the records arrive in. -test("the store writer merges the compatible ids of entries that share a schema id, in either order", async () => { - const entry = (compatibleSchemaIds: string[]) => ({ - schemaId: "tests.merge.x", - schemaName: "tests.merge", - schemaVersion: "tests.merge.v1", - contentHash: "tests.merge.x", - canonicalSchemaJson: "", - compatibleSchemaIds, - members: [], - }); - const record = (key: string, schemaId: string, compatibleSchemaIds: string[]) => ({ - key, - type: "tests.merge", - schemaId, - payload: encode({ key }), - storedAt: "2026-09-30T00:00:00.0000000Z", - catalogEntry: entry(compatibleSchemaIds), - }); - const a = record("a", "tests.merge.x", ["tests.merge.x"]); - const b = record("b", "tests.merge.y", ["tests.merge.x", "tests.merge.y"]); - const dir = await mkdtemp(join(tmpdir(), "cultcache-merge-")); +// The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever order +// the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id. +const writerEntry = (schemaId: string, schemaName: string, contentHash: string, compatibleSchemaIds: string[]) => ({ + schemaId, + schemaName, + schemaVersion: `${schemaName}.v1`, + contentHash, + canonicalSchemaJson: "", + compatibleSchemaIds, + members: [], +}); +const writerRecord = (key: string, type: string, schemaId: string, catalogEntry?: ReturnType) => ({ + key, + type, + schemaId, + payload: encode({ key }), + storedAt: "2026-09-30T00:00:00.0000000Z", + ...(catalogEntry ? { catalogEntry } : {}), +}); +const rawCatalogEntry = (entry: ReturnType) => [ + entry.schemaId, + entry.schemaName, + entry.schemaVersion, + entry.contentHash, + entry.canonicalSchemaJson, + entry.compatibleSchemaIds, + [], +]; +const catalogOf = async (file: string): Promise => + (decode(await readFile(file)) as [string, unknown[][], unknown[][]])[1]; - for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) { +test("a registered entry is written as it is, over the arrived entries with the same own id, in either order", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-registered-wins-")); + const stale1 = writerEntry("tests.x", "tests.n", "stale-1", ["tests.x", "old"]); + const stale2 = writerEntry("tests.x", "tests.n", "stale-2", ["tests.x"]); + const registered = writerRecord("a", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "fresh", ["tests.x"])); + + for (const [name, entries] of [["12", [stale1, stale2]], ["21", [stale2, stale1]]] as const) { const file = join(dir, `${name}.msgpack`); - await new SingleFileMessagePackBackingStore(file).pushAll([...order]); - const decoded = decode(await readFile(file)) as [string, unknown[][], unknown[][]]; - assert.deepEqual(decoded[1].map((catalogEntry) => (catalogEntry[5] as string[]).slice().sort()), [["tests.merge.x", "tests.merge.y"]], name); - assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2, name); + await writeFile( + file, + encode(["cultcache.store.v1", entries.map(rawCatalogEntry), [["b", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ b: 1 })]]]), + ); + await new SingleFileMessagePackBackingStore(file).push(registered); + const catalog = await catalogOf(file); + assert.equal(catalog.length, 1, name); + assert.equal(catalog[0]![3], "fresh", `${name}: the registered entry's hash is written`); + assert.deepEqual(catalog[0]![5], ["tests.x"], `${name}: written as chosen, no union with the arrived entries' ids`); } - - // A put over a key of a file whose other record is stored under the wider entry's compatible id. - const file = join(dir, "push.msgpack"); - await new SingleFileMessagePackBackingStore(file).pushAll([b]); - await new SingleFileMessagePackBackingStore(file).push(a); - assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2); }); -// Entries of different schemas that share an id are not merged: the record gets a default entry under its own id, so no record is -// read as another schema, whichever order the records arrive in. -test("the store writer never merges an entry of another schema into a record's catalog entry", async () => { - const entry = (schemaName: string, compatibleSchemaIds: string[]) => ({ - schemaId: "tests.retype.x", - schemaName, - schemaVersion: `${schemaName}.v1`, - contentHash: "tests.retype.x", - canonicalSchemaJson: "", - compatibleSchemaIds, - members: [], - }); - const record = (key: string, type: string, schemaId: string, catalogEntry: ReturnType) => ({ - key, - type, - schemaId, - payload: encode({ key }), - storedAt: "2026-09-30T00:00:00.0000000Z", - catalogEntry, - }); - // a is schema "mine" under x; b is schema "theirs" under y, carrying an entry with x's id that lists y. - const a = record("a", "tests.retype.mine", "tests.retype.x", entry("tests.retype.mine", ["tests.retype.x"])); - const b = record("b", "tests.retype.theirs", "tests.retype.y", entry("tests.retype.theirs", ["tests.retype.x", "tests.retype.y"])); +test("a rename with a stable schema id: a whole-view write names the registered schema", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-rename-")); + const file = join(dir, "rename.msgpack"); + await writeFile( + file, + encode(["cultcache.store.v1", [rawCatalogEntry(writerEntry("tests.x", "tests.old", "stale", ["tests.x"]))], [["a", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ a: 1 })]]]), + ); + await new SingleFileMessagePackBackingStore(file).pushAll([writerRecord("a", "tests.new", "tests.x", writerEntry("tests.x", "tests.new", "fresh", ["tests.x"]))]); + assert.deepEqual((await catalogOf(file)).map((entry) => [entry[1], entry[3]]), [["tests.new", "fresh"]]); +}); + +test("two entries of one tier that share an own id and disagree on the schema name refuse the write, typed", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-disagree-")); + const first = writerRecord("b", "tests.first", "tests.x", writerEntry("tests.x", "tests.first", "h1", ["tests.x"])); + const second = writerRecord("a", "tests.first", "tests.x", writerEntry("tests.x", "tests.second", "h2", ["tests.x"])); + for (const [name, order] of [["fs", [first, second]], ["sf", [second, first]]] as const) { + const file = join(dir, `${name}.msgpack`); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll([...order]), + (error) => + error instanceof SchemaConflictError && + error.schemaId === "tests.x" && + error.recordKey === "a" && + [...error.schemaNames].sort().join() === "tests.first,tests.second", + name, + ); + } +}); + +test("records of different types under one schema id refuse the write, typed, and no record changes type", async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-retype-")); + const a = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.x", "tests.n", "h1", ["tests.x", "tests.y"])); + const b = writerRecord("b", "tests.m", "tests.y", writerEntry("tests.y", "tests.m", "h2", ["tests.y"])); for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) { const file = join(dir, `${name}.msgpack`); - await new SingleFileMessagePackBackingStore(file).pushAll([...order]); - const back = await new SingleFileMessagePackBackingStore(file).pullAll(); - assert.deepEqual(back.map((envelope) => [envelope.key, envelope.type]).sort(), [["a", "tests.retype.mine"], ["b", "tests.retype.theirs"]], name); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll([...order]), + (error) => error instanceof SchemaConflictError && error.schemaId === "tests.y" && error.schemaNames.join() === "tests.m,tests.n", + name, + ); } }); +test("an entry that owns an id is chosen over one that lists it, and an entry no record needs is not written", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-own-first-")); + const owner = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.y", "tests.n", "h1", ["tests.y"])); + const lister = writerRecord("b", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "h2", ["tests.x", "tests.y"])); + const onlyOwner = join(dir, "own.msgpack"); + await new SingleFileMessagePackBackingStore(onlyOwner).pushAll([owner]); + assert.deepEqual((await catalogOf(onlyOwner)).map((entry) => entry[0]), ["tests.y"]); + + // The lister carried only its own id: its list of y is not needed, and the owner's entry publishes y. + for (const [name, order] of [["lo", [lister, owner]], ["ol", [owner, lister]]] as const) { + const file = join(dir, `${name}.msgpack`); + await new SingleFileMessagePackBackingStore(file).pushAll([...order]); + const catalog = await catalogOf(file); + assert.deepEqual(catalog.map((entry) => [entry[0], entry[5]]).sort(), [["tests.x", ["tests.x", "tests.y"]], ["tests.y", ["tests.y"]]], name); + } +}); + +test("a record no chosen entry publishes refuses the write, typed, and leaves the file", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-orphan-")); + const file = join(dir, "orphan.msgpack"); + // z sits under old, which only the arrived entry (id x) lists; the supplied entry owns x and does not. + await writeFile( + file, + encode([ + "cultcache.store.v1", + [rawCatalogEntry(writerEntry("tests.x", "tests.legacy", "stale", ["tests.x", "old"]))], + [["z", "old", "2026-09-30T00:00:00.0000000Z", encode({ z: 1 })]], + ]), + ); + const before = await readFile(file); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).push(writerRecord("a", "tests.legacy", "tests.x", writerEntry("tests.x", "tests.legacy", "fresh", ["tests.x"]))), + (error) => error instanceof SchemaConflictError && error.schemaId === "old" && error.recordKey === "z", + ); + assert.ok(before.equals(await readFile(file)), "the file is left as it was"); +}); + test("an id an entry owns names that entry, not one that lists it as compatible", async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-own-id-")); for (const vector of ["own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"]) { diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md new file mode 100644 index 00000000..cca85a79 --- /dev/null +++ b/packages/cultnet-py/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## Unreleased (breaking; version 0.x, so the next minor) + +- Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one + its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`, + which returns the document; take the schema id from `document.catalog_entry().schema_id`. +- `apply_raw_document_record`, `apply_raw_snapshot` and `apply_shard_log_response` report and stamp the local schema id, also for a record + that arrived under a compatible id or the schema name. diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 7063e3dc..4dee3a2f 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -251,18 +251,16 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentIndex[key] = ToIndexRecord(stored); } - var catalogCandidates = CultSchemaCatalogEntry.MergeById( - currentManifest.SchemaCatalog - .Concat(_durableCatalog) - .Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())), - preferLast: true) - .ToArray(); - var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal); - var targetCatalog = catalogCandidates - .Where(entry => usedSchemaIds.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(usedSchemaIds.Contains)) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + // A record this cache holds, unchanged since the manifest was read, is rewritten under its registered schema's id, as a dirty + // one is: the manifest entry that lists an older id for it is then not needed to publish it. + var restamped = currentIndex.Values + .Where(record => !_dirtyKeys.ContainsKey(record.Key) && + Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && + !string.Equals(known.Descriptor.SchemaId, record.SchemaId, StringComparison.Ordinal)) + .Select(record => record.Key) .ToArray(); var keysToWrite = _dirtyKeys.Keys + .Concat(restamped) .OrderBy(value => value, StringComparer.Ordinal) .ToArray(); foreach (var key in keysToWrite) @@ -285,6 +283,14 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore pagePayload); } + var targetCatalog = CultSchemaCatalogEntry.Derive( + currentIndex.Values, + Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), + currentManifest.SchemaCatalog.Concat(_durableCatalog).ToArray()) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .ToArray(); + // The directory store writes only its dirty pages, so it keeps a manifest already marked. var header = HeaderFor( keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds), diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 4fa024fe..ece39387 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -128,9 +128,20 @@ record with its registered schema's id, whatever id the record arrived under. A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and only when no entry owns the id to an entry that lists it as a compatible id (the first such entry -in catalog order). Two entries that share a `schemaId` are one schema seen by different writers: a -writer that merges them writes one entry whose compatible ids are the union of theirs, whatever -order they arrive in, so it lists every id any record it writes carries. +in catalog order). + +A writer derives the catalog from the records it writes, and from nothing else. For each distinct +schema id a record carries it chooses exactly one entry that publishes it: the entry that owns the id +(a registered schema's descriptor, else an entry the record arrived with), else an entry that lists +it as compatible (again registered first). When a registered descriptor and an arrived entry share an +own id, the descriptor wins: a rename or a stale content hash takes the descriptor's name, hash, +members and compatible ids as they are. Entries are written as chosen, deduplicated by own id, with +no union of compatible ids and no dependence on the order records arrive in; an entry no record needs +is not written. A write is refused, and the store left as it was, when two entries of one tier share +an own id and disagree on the schema name, when records of different types share a schema id, or when +a record's id is published by no chosen entry. Each runtime refuses with one typed error naming the +id, the schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError` +(Rust, TypeScript, Python). This is not decorative paperwork. It is what allows another CultCache implementation to inspect an old store, resolve the schema for each record, and diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 28884e10..18f5e19d 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -29,34 +29,92 @@ namespace GameCult.Caching public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); - // One entry per schema id, whatever order the candidates come in. Entries that share an id and a schema name are one schema seen - // by different writers: the merged entry takes its name, content hash and members from the entry being written now (the - // last with preferLast, else the first) and lists every id any of them lists. An entry of another schema under the same id is - // dropped, never merged: its ids are not this schema's. - internal static IEnumerable MergeById(IEnumerable entries, bool preferLast = false) + // The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one + // entry that publishes it: the entry that owns the id (a registered descriptor, else one that arrived with the records), else + // one that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered descriptor + // wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema names, or a + // record no chosen entry publishes, refuse the write. + // Entries that tie are taken in one fixed order, so the catalog does not depend on the order they arrive in. + private static IEnumerable Canonical(IEnumerable entries) => entries + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.ContentHash, StringComparer.Ordinal) + .ThenBy(entry => string.Join(",", entry.CompatibleSchemaIds), StringComparer.Ordinal); + + internal static CultSchemaCatalogEntry[] Derive( + IEnumerable records, + IReadOnlyCollection registered, + IReadOnlyCollection arrived) { - foreach (var group in entries.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)) + var chosen = new Dictionary(StringComparer.Ordinal); + var carried = records.GroupBy(record => record.SchemaId, StringComparer.Ordinal).OrderBy(group => group.Key, StringComparer.Ordinal).ToArray(); + foreach (var group in carried) { - var members = group.ToArray(); - var chosen = preferLast ? members[^1] : members[0]; - var sameSchema = members.Where(entry => string.Equals(entry.SchemaName, chosen.SchemaName, StringComparison.Ordinal)).ToArray(); - if (sameSchema.Length == 1) + var schemaId = group.Key; + var recordKey = group.Min(record => record.Key)!; + var ownRegisteredAll = Canonical(registered.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray(); + var ownRegistered = ownRegisteredAll.FirstOrDefault(); + var ownArrived = Canonical(arrived.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray(); + // Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + var disagreeing = ownRegisteredAll.Length > 0 ? ownRegisteredAll : ownArrived; + if (disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).Count() > 1) { - yield return chosen; - continue; + var names = disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).ToArray(); + throw new CultSchemaConflictException( + $"Schema id '{schemaId}' is claimed by schemas {string.Join(" and ", names.Select(name => $"'{name}'"))}; record '{recordKey}' cannot be written under it.", + schemaId, names, recordKey); } - yield return new CultSchemaCatalogEntry + CultSchemaCatalogEntry? pick; + bool isRegistered; + if (ownRegistered != null) { - SchemaId = chosen.SchemaId, - SchemaName = chosen.SchemaName, - SchemaVersion = chosen.SchemaVersion, - ContentHash = chosen.ContentHash, - CanonicalSchemaJson = chosen.CanonicalSchemaJson, - CompatibleSchemaIds = sameSchema.SelectMany(entry => entry.CompatibleSchemaIds).Distinct(StringComparer.Ordinal).ToArray(), - Members = chosen.Members - }; + pick = ownRegistered; + isRegistered = true; + } + else if (ownArrived.Length > 0) + { + pick = ownArrived[0]; + isRegistered = false; + } + else + { + pick = Canonical(registered.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + isRegistered = pick != null; + pick ??= Canonical(arrived.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + } + + if (pick == null) + continue; + if (chosen.TryGetValue(pick.SchemaId, out var prior)) + { + if (prior.Registered && !isRegistered) + continue; + + // Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other. + if (prior.Registered == isRegistered && !string.Equals(prior.Entry.SchemaName, pick.SchemaName, StringComparison.Ordinal)) + { + var names = new[] { prior.Entry.SchemaName, pick.SchemaName }; + throw new CultSchemaConflictException( + $"Schema id '{pick.SchemaId}' is claimed by schemas '{names[0]}' and '{names[1]}'; record '{recordKey}' cannot be written under it.", + pick.SchemaId, names, recordKey); + } + } + chosen[pick.SchemaId] = (pick, isRegistered); } + + foreach (var group in carried) + { + if (!chosen.Values.Any(pair => string.Equals(pair.Entry.SchemaId, group.Key, StringComparison.Ordinal) || + pair.Entry.CompatibleSchemaIds.Contains(group.Key, StringComparer.Ordinal))) + { + var recordKey = group.Min(record => record.Key)!; + throw new CultSchemaConflictException( + $"No schema entry publishes schema id '{group.Key}' of record '{recordKey}'.", + group.Key, chosen.Values.Select(pair => pair.Entry.SchemaName).ToArray(), recordKey); + } + } + + return chosen.Values.Select(pair => pair.Entry).ToArray(); } } @@ -3536,7 +3594,8 @@ namespace GameCult.Caching ReadSnapshot(); WriteSnapshot( Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), - Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()), + Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), + Array.Empty(), Entries.Values.Any(entry => entry.HoldsIds), existingHeader: null, wholeStore: true); @@ -3569,9 +3628,6 @@ namespace GameCult.Caching var records = ontoDisk ? disk.Records.ToDictionary(record => record.Key, StringComparer.Ordinal) : Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)).ToDictionary(record => record.Key, StringComparer.Ordinal); - var catalog = ontoDisk - ? disk.SchemaCatalog - : Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()); // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), // what they removed (departing), and this batch. @@ -3581,6 +3637,20 @@ namespace GameCult.Caching records.Remove(entry.Key.Value); foreach (var entry in request.Upserts) records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); + if (ontoDisk) + { + // A record this cache holds, unchanged since the file was read, is written under its registered schema's id, as a + // whole-view write writes it: the file's entry that lists an older id for it is then not needed to publish it. + foreach (var record in disk.Records) + { + if (records.TryGetValue(record.Key, out var carried) && ReferenceEquals(carried, record) && + Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && + !string.Equals(known.Descriptor.SchemaId, record.SchemaId, StringComparison.Ordinal)) + { + records[record.Key] = ToPersistedRecord(known, SerializePayload); + } + } + } // Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An // unconditional commit writes this store's whole view, so what its records hold decides. A record this commit @@ -3588,7 +3658,10 @@ namespace GameCult.Caching var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) || (!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value))); - WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); + var registered = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()) + .Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())) + .ToArray(); + WriteSnapshot(records.Values, registered, ontoDisk ? disk.SchemaCatalog : Array.Empty(), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); if (!ontoDisk) Cache?.IdsPersisted(records.Keys); foreach (var entry in request.Deletes) @@ -3660,17 +3733,15 @@ namespace GameCult.Caching } } - private void WriteSnapshot(IEnumerable records, IEnumerable catalog, bool holdsElementIds, string? existingHeader, bool wholeStore) + private void WriteSnapshot(IEnumerable records, IReadOnlyCollection registered, IReadOnlyCollection arrived, bool holdsElementIds, string? existingHeader, bool wholeStore) { var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); - var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); var snapshot = new CultPersistedStoreSnapshot { FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)), - SchemaCatalog = CultSchemaCatalogEntry.MergeById(catalog, preferLast: true) - // An entry stays while it publishes a schema id some record carries, as its own id or a compatible one. - .Where(entry => used.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(used.Contains)) + SchemaCatalog = CultSchemaCatalogEntry.Derive(ordered, registered, arrived) .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) .ToArray(), Records = ordered }; diff --git a/src/GameCult.Caching/CultSchemaConflictException.cs b/src/GameCult.Caching/CultSchemaConflictException.cs new file mode 100644 index 00000000..9419e2c5 --- /dev/null +++ b/src/GameCult.Caching/CultSchemaConflictException.cs @@ -0,0 +1,24 @@ +using System; + +namespace GameCult.Caching +{ + /// + /// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the + /// records, neither registered), or a record whose schema id no chosen entry publishes. Nothing is written, so the store stays as + /// it was. It names the id, the schema names involved and a record key. + /// + public sealed class CultSchemaConflictException : InvalidOperationException + { + public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey) + : base(message) + { + SchemaId = schemaId; + SchemaNames = schemaNames; + RecordKey = recordKey; + } + + public string SchemaId { get; } + public string[] SchemaNames { get; } + public string RecordKey { get; } + } +}