diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md new file mode 100644 index 00000000..70a59b76 --- /dev/null +++ b/packages/cultcache-py/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## Unreleased + +- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a + write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload. +- The single-file writer derives the catalog from the records it writes: one entry per carried schema id, the registered entry over an + arrived one, written as chosen and never merged. A cache stamps a record with its registered schema's id. diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index 84aaacf5..f0ae38b2 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -9,7 +9,7 @@ from .documents import ( define_document_registry, define_document_type, ) -from .stores import JsonLinesBackingStore, SingleFileMessagePackBackingStore, StoreUnreadableError +from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError __all__ = [ "BackingStore", @@ -21,6 +21,7 @@ __all__ = [ "DatabaseEntryField", "DocumentDefinition", "JsonLinesBackingStore", + "SchemaConflictError", "SingleFileMessagePackBackingStore", "StoreUnreadableError", "database_entry_field", diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index f06104b1..78de24f8 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -4,7 +4,6 @@ import base64 import json import os import threading -from dataclasses import replace from pathlib import Path from typing import Any @@ -34,6 +33,20 @@ class StoreUnreadableError(ValueError): self.path = path +class SchemaConflictError(ValueError): + """A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id + and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names + involved (`schema_names`) and a record key (`record_key`).""" + + def __init__(self, schema_id: str, schema_names: list[str], record_key: str) -> None: + super().__init__( + f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}" + ) + self.schema_id = schema_id + self.schema_names = schema_names + self.record_key = record_key + + class JsonLinesBackingStore: def __init__(self, path: str | os.PathLike[str]) -> None: self.path = Path(path) @@ -125,27 +138,21 @@ class SingleFileMessagePackBackingStore: def push(self, envelope: CultCacheEnvelope) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - existing[(envelope.type, envelope.key)] = envelope - self._replace_all(list(existing.values())) + self._replace_all([envelope], self.pull_all()) def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: with self._lock: - existing = {(item.type, item.key): item for item in self.pull_all()} - for envelope in envelopes: - existing[(envelope.type, envelope.key)] = envelope - self._replace_all(list(existing.values())) + self._replace_all(list(envelopes), self.pull_all()) def delete(self, type: str, key: str) -> None: with self._lock: - existing = [item for item in self.pull_all() if not (item.type == type and item.key == key)] - self._replace_all(existing) + self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)]) - def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: + def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: msgpack = self._msgpack() self.path.parent.mkdir(parents=True, exist_ok=True) temp = self.path.with_suffix(self.path.suffix + ".tmp") - temp.write_bytes(msgpack.packb(_encode_snapshot(envelopes, self._format), use_bin_type=True)) + temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) temp.replace(self.path) @staticmethod @@ -160,46 +167,81 @@ class SingleFileMessagePackBackingStore: return msgpack -def _encode_snapshot(envelopes: list[CultCacheEnvelope], format_version: str) -> list[Any]: - # Every record's schema id is published by a catalog entry: the entry the envelope carries when it publishes that id (as its - # own id or a compatible one), else a default entry under the record's id. The catalog is keyed by the entry's own id, so a - # record read under a compatible id is written back beside the entry that lists it. - catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {} +def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]: + """Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in.""" + return sorted(candidates, key=lambda entry: (entry.schema_name, entry.content_hash, ",".join(entry.compatible_schema_ids))) - def place(envelope: CultCacheEnvelope, candidates: list[CultCacheSchemaCatalogEntry]) -> None: - for candidate in candidates: - existing = catalog_by_schema_id.get(candidate.schema_id) - if existing is None: - catalog_by_schema_id[candidate.schema_id] = candidate - return - # Entries that share an id and a schema name are one schema seen by different writers: the entry written lists every id - # any of them lists. An entry of another schema is never merged: the record gets a default entry under its own id. - if existing.schema_name == candidate.schema_name: - catalog_by_schema_id[candidate.schema_id] = replace( - existing, - compatible_schema_ids=tuple(dict.fromkeys((*existing.compatible_schema_ids, *candidate.compatible_schema_ids))), - ) - return - raise ValueError(f"CultCache schema id {_schema_id_for(envelope)!r} cannot identify two different schemas") - # A record's own id is claimed first, so an entry that lists it as compatible cannot take it from the schema that owns it, - # whatever order the records arrive in. - listed: list[tuple[CultCacheEnvelope, CultCacheSchemaCatalogEntry]] = [] - for envelope in envelopes: +def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultCacheSchemaCatalogEntry]: + """The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, + one entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one + that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an + arrived one with the same own id. Records of different types under one id, two arrived entries with the same own id and + different schema names, or a record no chosen entry publishes, refuse the write. `records` are (envelope, registered): an + envelope the caller supplied is registered, one read back from the file is arrived.""" + entries: list[tuple[CultCacheSchemaCatalogEntry, bool]] = [] + by_id: dict[str, list[CultCacheEnvelope]] = {} + for envelope, registered in records: schema_id = _schema_id_for(envelope) supplied = envelope.catalog_entry - if supplied is not None and supplied.schema_id != schema_id and schema_id in supplied.compatible_schema_ids: - listed.append((envelope, supplied)) - elif supplied is not None and supplied.schema_id == schema_id: - place(envelope, [supplied]) - else: - place(envelope, [_default_catalog_entry(envelope)]) - for envelope, supplied in listed: - place(envelope, [supplied, _default_catalog_entry(envelope)]) + publishes = supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids) + entries.append((supplied if publishes else _default_catalog_entry(envelope), registered)) + by_id.setdefault(schema_id, []).append(envelope) + chosen: dict[str, tuple[CultCacheSchemaCatalogEntry, bool]] = {} + for schema_id in sorted(by_id): + carrying = by_id[schema_id] + record_key = min(envelope.key for envelope in carrying) + types = sorted({envelope.type for envelope in carrying}) + if len(types) > 1: + raise SchemaConflictError(schema_id, types, record_key) + own_registered = next(iter(canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)), None) + own_arrived = canonical(entry for entry, registered in entries if not registered and entry.schema_id == schema_id) + own_registered_all = canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id) + # Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + names = sorted({entry.schema_name for entry in (own_registered_all or own_arrived)}) + if len(names) > 1: + raise SchemaConflictError(schema_id, names, record_key) + if own_registered is not None: + pick, registered_pick = own_registered, True + elif own_arrived: + pick, registered_pick = own_arrived[0], False + else: + listed_registered = next(iter(canonical(entry for entry, registered in entries if registered and schema_id in entry.compatible_schema_ids)), None) + listed_arrived = next(iter(canonical(entry for entry, registered in entries if not registered and schema_id in entry.compatible_schema_ids)), None) + pick = listed_registered if listed_registered is not None else listed_arrived + registered_pick = listed_registered is not None + if pick is None: + continue + prior = chosen.get(pick.schema_id) + if prior is not None: + if prior[1] and not registered_pick: + continue + # Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other. + if prior[1] == registered_pick and prior[0].schema_name != pick.schema_name: + raise SchemaConflictError(pick.schema_id, sorted({prior[0].schema_name, pick.schema_name}), record_key) + chosen[pick.schema_id] = (pick, registered_pick) + + for schema_id in sorted(by_id): + if not any(entry.schema_id == schema_id or schema_id in entry.compatible_schema_ids for entry, _ in chosen.values()): + raise SchemaConflictError( + schema_id, sorted(entry.schema_name for entry, _ in chosen.values()), min(envelope.key for envelope in by_id[schema_id]) + ) + return [entry for entry, _ in chosen.values()] + + +def _encode_snapshot( + supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str +) -> list[Any]: + replaced = {(envelope.type, envelope.key) for envelope in supplied} + kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced] + envelopes = [*kept, *supplied] catalog = [ _encode_catalog_entry(entry) - for entry in sorted(catalog_by_schema_id.values(), key=lambda item: item.schema_name) + for entry in sorted( + _derive_catalog([*((envelope, False) for envelope in kept), *((envelope, True) for envelope in supplied)]), + key=lambda item: (item.schema_name, item.schema_id), + ) ] records = [ [envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload] diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index ffbc2706..408afd6c 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -315,68 +315,130 @@ class CultCacheTests(unittest.TestCase): self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"]) self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1) - # Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists, - # whichever order the records arrive in. - def test_the_store_writer_merges_the_compatible_ids_of_entries_that_share_a_schema_id_in_either_order(self) -> None: - import msgpack # type: ignore + # The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever + # order the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id. + @staticmethod + def _writer_entry(schema_id: str, name: str, content_hash: str, compatible: tuple[str, ...]): from cultcache_py import CultCacheSchemaCatalogEntry - from cultcache_py.stores import _encode_snapshot - def entry(compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry: - return CultCacheSchemaCatalogEntry( - schema_id="tests.merge.x", schema_name="tests.merge", schema_version="tests.merge.v1", - content_hash="tests.merge.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(), - ) + return CultCacheSchemaCatalogEntry( + schema_id=schema_id, schema_name=name, schema_version=name + ".v1", content_hash=content_hash, + canonical_schema_json="", compatible_schema_ids=compatible, members=(), + ) - def record(key: str, schema_id: str, compatible: tuple[str, ...]) -> CultCacheEnvelope: - return CultCacheEnvelope( - key=key, type="tests.merge", payload=msgpack.packb({"key": key}, use_bin_type=True), - stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=entry(compatible), - ) + def _writer_record(self, key: str, type: str, schema_id: str, catalog_entry=None) -> CultCacheEnvelope: + import msgpack # type: ignore - a = record("a", "tests.merge.x", ("tests.merge.x",)) - b = record("b", "tests.merge.y", ("tests.merge.x", "tests.merge.y")) - for order in ([a, b], [b, a]): - catalog = _encode_snapshot(order, "cultcache.store.v1")[1] - self.assertEqual([sorted(entry[5]) for entry in catalog], [["tests.merge.x", "tests.merge.y"]]) + return CultCacheEnvelope( + key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True), + stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry, + ) + + @staticmethod + def _raw_entry(entry) -> list: + return [entry.schema_id, entry.schema_name, entry.schema_version, entry.content_hash, "", list(entry.compatible_schema_ids), []] + + def test_a_registered_entry_is_written_as_it_is_over_the_arrived_entries_with_the_same_own_id_in_either_order(self) -> None: + import msgpack # type: ignore + + stale1 = self._writer_entry("tests.x", "tests.n", "stale-1", ("tests.x", "old")) + stale2 = self._writer_entry("tests.x", "tests.n", "stale-2", ("tests.x",)) + registered = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "fresh", ("tests.x",))) + for entries in ([stale1, stale2], [stale2, stale1]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(entry) for entry in entries], + [["b", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"b": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push(registered) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual(len(catalog), 1) + self.assertEqual(catalog[0][3], "fresh") + self.assertEqual(catalog[0][5], ["tests.x"], "written as chosen, no union with the arrived entries' ids") + + def test_a_rename_with_a_stable_schema_id_a_whole_view_write_names_the_registered_schema(self) -> None: + import msgpack # type: ignore with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "store.msgpack" - SingleFileMessagePackBackingStore(path).push_all([b]) - SingleFileMessagePackBackingStore(path).push(a) - self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2) + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.old", "stale", ("tests.x",)))], + [["a", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"a": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + SingleFileMessagePackBackingStore(path).push_all( + [self._writer_record("a", "tests.new", "tests.x", self._writer_entry("tests.x", "tests.new", "fresh", ("tests.x",)))] + ) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual([(entry[1], entry[3]) for entry in catalog], [("tests.new", "fresh")]) + + def test_two_entries_of_one_tier_that_share_an_own_id_and_disagree_on_the_schema_name_refuse_the_write_typed(self) -> None: + from cultcache_py import SchemaConflictError + + first = self._writer_record("b", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.first", "h1", ("tests.x",))) + second = self._writer_record("a", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.second", "h2", ("tests.x",))) + for order in ([first, second], [second, first]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.x") + self.assertEqual(refused.exception.record_key, "a") + self.assertEqual(refused.exception.schema_names, ["tests.first", "tests.second"]) + self.assertFalse(path.exists()) + + def test_records_of_different_types_under_one_schema_id_refuse_the_write_typed_and_no_record_changes_type(self) -> None: + from cultcache_py import SchemaConflictError + + a = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y"))) + b = self._writer_record("b", "tests.m", "tests.y", self._writer_entry("tests.y", "tests.m", "h2", ("tests.y",))) + for order in ([a, b], [b, a]): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order) + self.assertEqual(refused.exception.schema_id, "tests.y") + self.assertEqual(refused.exception.schema_names, ["tests.m", "tests.n"]) + + def test_an_entry_that_owns_an_id_is_chosen_over_one_that_lists_it(self) -> None: + import msgpack # type: ignore + + owner = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.y", "tests.n", "h1", ("tests.y",))) + lister = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.y"))) + for order in ([lister, owner], [owner, lister]): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all(order) + catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1] + self.assertEqual(sorted((entry[0], entry[5]) for entry in catalog), [("tests.x", ["tests.x", "tests.y"]), ("tests.y", ["tests.y"])]) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + SingleFileMessagePackBackingStore(path).push_all([owner]) + self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.y"]) + + def test_a_record_no_chosen_entry_publishes_refuses_the_write_typed_and_leaves_the_file(self) -> None: + import msgpack # type: ignore + from cultcache_py import SchemaConflictError + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.msgpack" + path.write_bytes(msgpack.packb([ + "cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.legacy", "stale", ("tests.x", "old")))], + [["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]], + ], use_bin_type=True)) + before = path.read_bytes() + with self.assertRaises(SchemaConflictError) as refused: + SingleFileMessagePackBackingStore(path).push( + self._writer_record("a", "tests.legacy", "tests.x", self._writer_entry("tests.x", "tests.legacy", "fresh", ("tests.x",))) + ) + self.assertEqual(refused.exception.schema_id, "old") + self.assertEqual(refused.exception.record_key, "z") + self.assertEqual(path.read_bytes(), before) def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None: for vector in ("own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"): store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / vector) self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"], vector) - # Entries of different schemas that share an id are not merged: the record gets a default entry under its own id, so no record - # is read as another schema, whichever order the records arrive in. - def test_the_store_writer_never_merges_an_entry_of_another_schema(self) -> None: - import msgpack # type: ignore - from cultcache_py import CultCacheSchemaCatalogEntry - from cultcache_py.stores import _decode_snapshot, _encode_snapshot - - def entry(name: str, compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry: - return CultCacheSchemaCatalogEntry( - schema_id="tests.retype.x", schema_name=name, schema_version=name + ".v1", - content_hash="tests.retype.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(), - ) - - def record(key: str, type: str, schema_id: str, catalog_entry: CultCacheSchemaCatalogEntry) -> CultCacheEnvelope: - return CultCacheEnvelope( - key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True), - stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry, - ) - - a = record("a", "tests.retype.mine", "tests.retype.x", entry("tests.retype.mine", ("tests.retype.x",))) - b = record("b", "tests.retype.theirs", "tests.retype.y", entry("tests.retype.theirs", ("tests.retype.x", "tests.retype.y"))) - for order in ([a, b], [b, a]): - decoded = msgpack.unpackb(msgpack.packb(_encode_snapshot(order, "cultcache.store.v1"), use_bin_type=True), raw=False) - _, envelopes = _decode_snapshot(decoded) - self.assertEqual(sorted((e.key, e.type) for e in envelopes), [("a", "tests.retype.mine"), ("b", "tests.retype.theirs")]) - def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None: import msgpack # type: ignore diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 0be95fed..fbc905cf 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -389,6 +389,27 @@ impl std::error::Error for StoreUnreadableError { } } +/// A write the catalog cannot describe: records of different types under one schema id. This writer derives one entry per carried +/// schema id from its records, so that is its only conflict. Nothing is written. It names the id, the two names and a record key. +#[derive(Debug)] +pub struct SchemaConflictError { + pub schema_id: String, + pub schema_names: Vec, + pub record_key: String, +} + +impl std::fmt::Display for SchemaConflictError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "schema id {:?} cannot describe record {:?}: {:?}", + self.schema_id, self.record_key, self.schema_names + ) + } +} + +impl std::error::Error for SchemaConflictError {} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct PushAllOptions { pub soft: bool, @@ -2592,11 +2613,13 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope], format: &str) -> Result< .clone() .unwrap_or_else(|| entry.r#type.clone()); if let Some(existing_type) = schema_types.insert(schema_id.clone(), entry.r#type.clone()) { - ensure!( - existing_type == entry.r#type, - "CultCache schema {schema_id:?} cannot identify both {existing_type:?} and {:?}", - entry.r#type - ); + if existing_type != entry.r#type { + return Err(anyhow::Error::new(SchemaConflictError { + schema_id, + schema_names: vec![existing_type, entry.r#type.clone()], + record_key: entry.key.clone(), + })); + } } } @@ -4842,6 +4865,33 @@ mod tests { Ok(()) } + // Records of different types under one schema id cannot be described by one catalog entry: the write is refused, typed, in + // either order, and the file is left as it was. A rename with a stable id (same type name in the new schema) writes fine. + #[test] + fn records_of_different_types_under_one_schema_id_refuse_the_write_typed() -> Result<()> { + let record = |key: &str, r#type: &str| CultCacheEnvelope { + key: key.into(), + r#type: r#type.into(), + payload: b"one".to_vec(), + stored_at: "2026-09-30T00:00:00Z".into(), + schema_id: Some("tests.x".into()), + }; + for order in [vec![record("a", "tests.n"), record("b", "tests.m")], vec![record("b", "tests.m"), record("a", "tests.n")]] { + let temp = tempfile::tempdir()?; + let path = temp.path().join("store.msgpack"); + let mut store = SingleFileMessagePackBackingStore::new(&path); + let error = store.push_all(&order, PushAllOptions::default()).unwrap_err(); + let conflict = error.downcast_ref::().unwrap_or_else(|| panic!("{error:#}")); + assert_eq!(conflict.schema_id, "tests.x"); + let mut names = conflict.schema_names.clone(); + names.sort(); + assert_eq!(names, vec!["tests.m".to_string(), "tests.n".to_string()]); + assert!(conflict.record_key == "a" || conflict.record_key == "b"); + assert!(!path.exists(), "nothing is written"); + } + Ok(()) + } + // A store cut short is refused whatever its version, and left as it is. #[test] fn a_truncated_v1_or_v2_store_is_refused_on_open_and_rewrite() -> Result<()> { diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 4f1388e1..a2f45d53 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -2,7 +2,7 @@ export { CultCache, CultCacheBuilder } from "./cult-cache"; export { inspectCultCacheBytes } from "./cult-cache-inspector"; export { defineDocumentRegistry, defineDocumentType } from "./document"; export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store"; -export { StoreUnreadableError } from "./store-format"; +export { SchemaConflictError, StoreUnreadableError } from "./store-format"; export type { AnyCultCacheDocumentDefinition, CacheBackingStore, diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index b8227ba2..23fa9ab4 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -4,7 +4,7 @@ import { dirname, resolve } from "node:path"; import { decode, encode } from "@msgpack/msgpack"; import { z } from "zod"; -import { STORE_FORMAT_VERSION, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format"; +import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format"; import type { CacheBackingStore, CultCacheEnvelope, @@ -49,7 +49,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const disk = await this.#readDisk(); this.#format = disk.format; if (disk.repairedLegacyPayload) { - await this.#writeAll(disk.envelopes); + await this.#writeAll([], disk.envelopes); } return disk.envelopes; @@ -61,8 +61,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const filtered = existing.filter( (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), ); - filtered.push(entry); - await this.#writeAll(filtered); + await this.#writeAll([entry], filtered); }); } @@ -72,7 +71,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { const filtered = existing.filter( (candidate) => !(candidate.type === entry.type && candidate.key === entry.key), ); - await this.#writeAll(filtered); + await this.#writeAll([], filtered); }); } @@ -95,7 +94,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { // reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused // and left as it is. this.#format = (await this.#readDisk()).format; - await this.#writeAll(entries); + await this.#writeAll(entries, []); }); } @@ -140,7 +139,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { return result; } - async #writeAll(entries: CultCacheEnvelope[]): Promise { + async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise { await mkdir(dirname(this.filePath), { recursive: true }); const tempPath = `${this.filePath}.tmp-${process.pid}-${Date.now()}-${Math.random() @@ -148,7 +147,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { .slice(2)}`; try { - await writeFile(tempPath, encode(encodeSnapshot(entries, this.#format))); + await writeFile(tempPath, encode(encodeSnapshot(supplied, arrived, this.#format))); await renameWithRetry(tempPath, this.filePath); } catch (error) { await rm(tempPath, { force: true }).catch(() => undefined); @@ -196,9 +195,14 @@ type DecodedSnapshot = { records: PersistedRecord[]; }; -function encodeSnapshot(entries: CultCacheEnvelope[], format: StoreFormat): unknown[] { - const catalog = [...catalogEntriesFor(entries).values()] - .sort((left, right) => compareOrdinal(left.schemaName, right.schemaName)); +function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] { + const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`)); + const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`)); + const entries = [...kept, ...supplied]; + const catalog = catalogEntriesFor([ + ...kept.map((envelope) => ({ envelope, registered: false })), + ...supplied.map((envelope) => ({ envelope, registered: true })), + ]).sort((left, right) => compareOrdinal(left.schemaName, right.schemaName) || compareOrdinal(left.schemaId, right.schemaId)); const records = [...entries] .sort((left, right) => compareOrdinal(left.key, right.key)) .map((entry) => [ @@ -215,71 +219,113 @@ function encodeSnapshot(entries: CultCacheEnvelope[], format: StoreFormat): unkn ]; } -// Every record's schema id is published by a catalog entry: the entry the envelope carries when it publishes that id (as its -// own id or a compatible one), else a default entry under the record's id. The catalog is keyed by the entry's own id, so a -// record read under a compatible id is written back beside the entry that lists it. -function catalogEntriesFor(entries: CultCacheEnvelope[]): Map { - const catalog = new Map(); - const defaultEntryFor = (entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry => { - const schemaId = schemaIdForEnvelope(entry); - return { - schemaId, +// The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one +// entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one that +// lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an arrived +// one with the same own id. Records of different types under one id, two arrived entries with the same own id and different schema +// names, or a record no chosen entry publishes, refuse the write. An envelope the caller supplied is registered; one read back +// from the file is arrived. +function catalogEntriesFor( + records: Array<{ envelope: CultCacheEnvelope; registered: boolean }>, +): CultCacheSchemaCatalogEntry[] { + const entries: Array<{ entry: CultCacheSchemaCatalogEntry; registered: boolean }> = []; + const byId = new Map(); + for (const { envelope, registered } of records) { + const schemaId = schemaIdForEnvelope(envelope); + const supplied = envelope.catalogEntry; + const publishes = + supplied !== undefined && (supplied.schemaId === schemaId || (supplied.compatibleSchemaIds ?? []).includes(schemaId)); + entries.push({ entry: publishes && supplied ? supplied : defaultEntryFor(envelope), registered }); + byId.set(schemaId, [...(byId.get(schemaId) ?? []), envelope]); + } + + const lists = (entry: CultCacheSchemaCatalogEntry, schemaId: string): boolean => + (entry.compatibleSchemaIds ?? [entry.schemaId]).includes(schemaId); + const chosen = new Map(); + for (const schemaId of [...byId.keys()].sort(compareOrdinal)) { + const carrying = byId.get(schemaId)!; + const recordKey = carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!; + const types = [...new Set(carrying.map((envelope) => envelope.type))].sort(compareOrdinal); + if (types.length > 1) { + throw new SchemaConflictError(schemaId, types, recordKey); + } + + const ownRegistered = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId))[0]; + const ownArrived = canonical(entries.filter((candidate) => !candidate.registered && candidate.entry.schemaId === schemaId)); + // Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + const ownRegisteredAll = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId)); + const names = [ + ...new Set((ownRegisteredAll.length > 0 ? ownRegisteredAll : ownArrived).map((candidate) => candidate.entry.schemaName)), + ].sort(compareOrdinal); + if (names.length > 1) { + throw new SchemaConflictError(schemaId, names, recordKey); + } + + let pick: { entry: CultCacheSchemaCatalogEntry; registered: boolean } | undefined; + if (ownRegistered !== undefined) { + pick = ownRegistered; + } else if (ownArrived.length > 0) { + pick = ownArrived[0]; + } else { + pick = + canonical(entries.filter((candidate) => candidate.registered && lists(candidate.entry, schemaId)))[0] ?? + canonical(entries.filter((candidate) => !candidate.registered && lists(candidate.entry, schemaId)))[0]; + } + + if (pick === undefined) { + continue; + } + + const prior = chosen.get(pick.entry.schemaId); + if (prior !== undefined) { + if (prior.registered && !pick.registered) { + continue; + } + + // Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other. + if (prior.registered === pick.registered && prior.entry.schemaName !== pick.entry.schemaName) { + throw new SchemaConflictError(pick.entry.schemaId, [prior.entry.schemaName, pick.entry.schemaName].sort(compareOrdinal), recordKey); + } + } + + chosen.set(pick.entry.schemaId, pick); + } + + for (const [schemaId, carrying] of byId) { + if (![...chosen.values()].some(({ entry }) => entry.schemaId === schemaId || lists(entry, schemaId))) { + throw new SchemaConflictError( + schemaId, + [...chosen.values()].map(({ entry }) => entry.schemaName).sort(compareOrdinal), + carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!, + ); + } + } + + return [...chosen.values()].map(({ entry }) => entry); +} + +// Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in. +function canonical(candidates: T[]): T[] { + const key = (candidate: T): string => + [candidate.entry.schemaName, candidate.entry.contentHash ?? "", (candidate.entry.compatibleSchemaIds ?? []).join(",")].join("\u0000"); + return [...candidates].sort((left, right) => compareOrdinal(key(left), key(right))); +} + +function defaultEntryFor(entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry { + const schemaId = schemaIdForEnvelope(entry); + return { + schemaId, + schemaName: entry.type, + schemaVersion: `${entry.type}.v1`, + contentHash: schemaId, + canonicalSchemaJson: JSON.stringify({ schemaName: entry.type, schemaVersion: `${entry.type}.v1`, - contentHash: schemaId, - canonicalSchemaJson: JSON.stringify({ - schemaName: entry.type, - schemaVersion: `${entry.type}.v1`, - members: [], - }), - compatibleSchemaIds: [schemaId], members: [], - }; + }), + compatibleSchemaIds: [schemaId], + members: [], }; - const place = (entry: CultCacheEnvelope, candidates: CultCacheSchemaCatalogEntry[]): void => { - for (const candidate of candidates) { - const existing = catalog.get(candidate.schemaId); - if (existing === undefined) { - catalog.set(candidate.schemaId, candidate); - return; - } - - // Entries that share an id and a schema name are one schema seen by different writers: the entry written lists every id any - // of them lists. An entry of another schema is never merged: the record gets a default entry under its own id. - if (existing.schemaName === candidate.schemaName) { - catalog.set(candidate.schemaId, { - ...existing, - compatibleSchemaIds: [ - ...new Set([...(existing.compatibleSchemaIds ?? [existing.schemaId]), ...(candidate.compatibleSchemaIds ?? [candidate.schemaId])]), - ], - }); - return; - } - } - - throw new Error(`CultCache schema id "${schemaIdForEnvelope(entry)}" cannot identify two different schemas.`); - }; - - // A record's own id is claimed first, so an entry that lists it as compatible cannot take it from the schema that owns it, - // whatever order the records arrive in. - const listed: Array<[CultCacheEnvelope, CultCacheSchemaCatalogEntry]> = []; - for (const entry of entries) { - const schemaId = schemaIdForEnvelope(entry); - const supplied = entry.catalogEntry; - if (supplied !== undefined && supplied.schemaId !== schemaId && (supplied.compatibleSchemaIds ?? []).includes(schemaId)) { - listed.push([entry, supplied]); - } else if (supplied !== undefined && supplied.schemaId === schemaId) { - place(entry, [supplied]); - } else { - place(entry, [defaultEntryFor(entry)]); - } - } - - for (const [entry, supplied] of listed) { - place(entry, [supplied, defaultEntryFor(entry)]); - } - - return catalog; } function schemaIdForEnvelope(entry: CultCacheEnvelope): string { diff --git a/packages/cultcache-ts/src/store-format.ts b/packages/cultcache-ts/src/store-format.ts index 9c0a6fa5..37d12761 100644 --- a/packages/cultcache-ts/src/store-format.ts +++ b/packages/cultcache-ts/src/store-format.ts @@ -32,6 +32,25 @@ export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...u return true; } +/** + * A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id and + * disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names involved and + * a record key. + */ +export class SchemaConflictError extends Error { + readonly schemaId: string; + readonly schemaNames: string[]; + readonly recordKey: string; + + constructor(schemaId: string, schemaNames: string[], recordKey: string) { + super(`Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`); + this.name = "SchemaConflictError"; + this.schemaId = schemaId; + this.schemaNames = schemaNames; + this.recordKey = recordKey; + } +} + /** * A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra slot), * a header or record it does not know, or a body it cannot decode. Open, flush and every rewrite refuse a file with this diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 21b37e1b..76836ead 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -15,7 +15,7 @@ import { CultCache } from "../src/cult-cache"; import { inspectCultCacheBytes } from "../src/cult-cache-inspector"; import { defineDocumentRegistry, defineDocumentType } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; -import { StoreUnreadableError } from "../src/store-format"; +import { SchemaConflictError, StoreUnreadableError } from "../src/store-format"; import type { CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types"; const execFileAsync = promisify(execFile); @@ -1344,78 +1344,138 @@ test("the store publishes a record's schema id even when the envelope's catalog assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 1); }); -// Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists, -// whichever order the records arrive in. -test("the store writer merges the compatible ids of entries that share a schema id, in either order", async () => { - const entry = (compatibleSchemaIds: string[]) => ({ - schemaId: "tests.merge.x", - schemaName: "tests.merge", - schemaVersion: "tests.merge.v1", - contentHash: "tests.merge.x", - canonicalSchemaJson: "", - compatibleSchemaIds, - members: [], - }); - const record = (key: string, schemaId: string, compatibleSchemaIds: string[]) => ({ - key, - type: "tests.merge", - schemaId, - payload: encode({ key }), - storedAt: "2026-09-30T00:00:00.0000000Z", - catalogEntry: entry(compatibleSchemaIds), - }); - const a = record("a", "tests.merge.x", ["tests.merge.x"]); - const b = record("b", "tests.merge.y", ["tests.merge.x", "tests.merge.y"]); - const dir = await mkdtemp(join(tmpdir(), "cultcache-merge-")); +// The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever order +// the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id. +const writerEntry = (schemaId: string, schemaName: string, contentHash: string, compatibleSchemaIds: string[]) => ({ + schemaId, + schemaName, + schemaVersion: `${schemaName}.v1`, + contentHash, + canonicalSchemaJson: "", + compatibleSchemaIds, + members: [], +}); +const writerRecord = (key: string, type: string, schemaId: string, catalogEntry?: ReturnType) => ({ + key, + type, + schemaId, + payload: encode({ key }), + storedAt: "2026-09-30T00:00:00.0000000Z", + ...(catalogEntry ? { catalogEntry } : {}), +}); +const rawCatalogEntry = (entry: ReturnType) => [ + entry.schemaId, + entry.schemaName, + entry.schemaVersion, + entry.contentHash, + entry.canonicalSchemaJson, + entry.compatibleSchemaIds, + [], +]; +const catalogOf = async (file: string): Promise => + (decode(await readFile(file)) as [string, unknown[][], unknown[][]])[1]; - for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) { +test("a registered entry is written as it is, over the arrived entries with the same own id, in either order", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-registered-wins-")); + const stale1 = writerEntry("tests.x", "tests.n", "stale-1", ["tests.x", "old"]); + const stale2 = writerEntry("tests.x", "tests.n", "stale-2", ["tests.x"]); + const registered = writerRecord("a", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "fresh", ["tests.x"])); + + for (const [name, entries] of [["12", [stale1, stale2]], ["21", [stale2, stale1]]] as const) { const file = join(dir, `${name}.msgpack`); - await new SingleFileMessagePackBackingStore(file).pushAll([...order]); - const decoded = decode(await readFile(file)) as [string, unknown[][], unknown[][]]; - assert.deepEqual(decoded[1].map((catalogEntry) => (catalogEntry[5] as string[]).slice().sort()), [["tests.merge.x", "tests.merge.y"]], name); - assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2, name); + await writeFile( + file, + encode(["cultcache.store.v1", entries.map(rawCatalogEntry), [["b", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ b: 1 })]]]), + ); + await new SingleFileMessagePackBackingStore(file).push(registered); + const catalog = await catalogOf(file); + assert.equal(catalog.length, 1, name); + assert.equal(catalog[0]![3], "fresh", `${name}: the registered entry's hash is written`); + assert.deepEqual(catalog[0]![5], ["tests.x"], `${name}: written as chosen, no union with the arrived entries' ids`); } - - // A put over a key of a file whose other record is stored under the wider entry's compatible id. - const file = join(dir, "push.msgpack"); - await new SingleFileMessagePackBackingStore(file).pushAll([b]); - await new SingleFileMessagePackBackingStore(file).push(a); - assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2); }); -// Entries of different schemas that share an id are not merged: the record gets a default entry under its own id, so no record is -// read as another schema, whichever order the records arrive in. -test("the store writer never merges an entry of another schema into a record's catalog entry", async () => { - const entry = (schemaName: string, compatibleSchemaIds: string[]) => ({ - schemaId: "tests.retype.x", - schemaName, - schemaVersion: `${schemaName}.v1`, - contentHash: "tests.retype.x", - canonicalSchemaJson: "", - compatibleSchemaIds, - members: [], - }); - const record = (key: string, type: string, schemaId: string, catalogEntry: ReturnType) => ({ - key, - type, - schemaId, - payload: encode({ key }), - storedAt: "2026-09-30T00:00:00.0000000Z", - catalogEntry, - }); - // a is schema "mine" under x; b is schema "theirs" under y, carrying an entry with x's id that lists y. - const a = record("a", "tests.retype.mine", "tests.retype.x", entry("tests.retype.mine", ["tests.retype.x"])); - const b = record("b", "tests.retype.theirs", "tests.retype.y", entry("tests.retype.theirs", ["tests.retype.x", "tests.retype.y"])); +test("a rename with a stable schema id: a whole-view write names the registered schema", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-rename-")); + const file = join(dir, "rename.msgpack"); + await writeFile( + file, + encode(["cultcache.store.v1", [rawCatalogEntry(writerEntry("tests.x", "tests.old", "stale", ["tests.x"]))], [["a", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ a: 1 })]]]), + ); + await new SingleFileMessagePackBackingStore(file).pushAll([writerRecord("a", "tests.new", "tests.x", writerEntry("tests.x", "tests.new", "fresh", ["tests.x"]))]); + assert.deepEqual((await catalogOf(file)).map((entry) => [entry[1], entry[3]]), [["tests.new", "fresh"]]); +}); + +test("two entries of one tier that share an own id and disagree on the schema name refuse the write, typed", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-disagree-")); + const first = writerRecord("b", "tests.first", "tests.x", writerEntry("tests.x", "tests.first", "h1", ["tests.x"])); + const second = writerRecord("a", "tests.first", "tests.x", writerEntry("tests.x", "tests.second", "h2", ["tests.x"])); + for (const [name, order] of [["fs", [first, second]], ["sf", [second, first]]] as const) { + const file = join(dir, `${name}.msgpack`); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll([...order]), + (error) => + error instanceof SchemaConflictError && + error.schemaId === "tests.x" && + error.recordKey === "a" && + [...error.schemaNames].sort().join() === "tests.first,tests.second", + name, + ); + } +}); + +test("records of different types under one schema id refuse the write, typed, and no record changes type", async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-retype-")); + const a = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.x", "tests.n", "h1", ["tests.x", "tests.y"])); + const b = writerRecord("b", "tests.m", "tests.y", writerEntry("tests.y", "tests.m", "h2", ["tests.y"])); for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) { const file = join(dir, `${name}.msgpack`); - await new SingleFileMessagePackBackingStore(file).pushAll([...order]); - const back = await new SingleFileMessagePackBackingStore(file).pullAll(); - assert.deepEqual(back.map((envelope) => [envelope.key, envelope.type]).sort(), [["a", "tests.retype.mine"], ["b", "tests.retype.theirs"]], name); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).pushAll([...order]), + (error) => error instanceof SchemaConflictError && error.schemaId === "tests.y" && error.schemaNames.join() === "tests.m,tests.n", + name, + ); } }); +test("an entry that owns an id is chosen over one that lists it, and an entry no record needs is not written", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-own-first-")); + const owner = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.y", "tests.n", "h1", ["tests.y"])); + const lister = writerRecord("b", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "h2", ["tests.x", "tests.y"])); + const onlyOwner = join(dir, "own.msgpack"); + await new SingleFileMessagePackBackingStore(onlyOwner).pushAll([owner]); + assert.deepEqual((await catalogOf(onlyOwner)).map((entry) => entry[0]), ["tests.y"]); + + // The lister carried only its own id: its list of y is not needed, and the owner's entry publishes y. + for (const [name, order] of [["lo", [lister, owner]], ["ol", [owner, lister]]] as const) { + const file = join(dir, `${name}.msgpack`); + await new SingleFileMessagePackBackingStore(file).pushAll([...order]); + const catalog = await catalogOf(file); + assert.deepEqual(catalog.map((entry) => [entry[0], entry[5]]).sort(), [["tests.x", ["tests.x", "tests.y"]], ["tests.y", ["tests.y"]]], name); + } +}); + +test("a record no chosen entry publishes refuses the write, typed, and leaves the file", async () => { + const dir = await mkdtemp(join(tmpdir(), "cultcache-orphan-")); + const file = join(dir, "orphan.msgpack"); + // z sits under old, which only the arrived entry (id x) lists; the supplied entry owns x and does not. + await writeFile( + file, + encode([ + "cultcache.store.v1", + [rawCatalogEntry(writerEntry("tests.x", "tests.legacy", "stale", ["tests.x", "old"]))], + [["z", "old", "2026-09-30T00:00:00.0000000Z", encode({ z: 1 })]], + ]), + ); + const before = await readFile(file); + await assert.rejects( + () => new SingleFileMessagePackBackingStore(file).push(writerRecord("a", "tests.legacy", "tests.x", writerEntry("tests.x", "tests.legacy", "fresh", ["tests.x"]))), + (error) => error instanceof SchemaConflictError && error.schemaId === "old" && error.recordKey === "z", + ); + assert.ok(before.equals(await readFile(file)), "the file is left as it was"); +}); + test("an id an entry owns names that entry, not one that lists it as compatible", async () => { const dir = await mkdtemp(join(tmpdir(), "cultcache-own-id-")); for (const vector of ["own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"]) { diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md new file mode 100644 index 00000000..cca85a79 --- /dev/null +++ b/packages/cultnet-py/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## Unreleased (breaking; version 0.x, so the next minor) + +- Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one + its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`, + which returns the document; take the schema id from `document.catalog_entry().schema_id`. +- `apply_raw_document_record`, `apply_raw_snapshot` and `apply_shard_log_response` report and stamp the local schema id, also for a record + that arrived under a compatible id or the schema name. diff --git a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs index 7063e3dc..4dee3a2f 100644 --- a/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs +++ b/src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs @@ -251,18 +251,16 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore currentIndex[key] = ToIndexRecord(stored); } - var catalogCandidates = CultSchemaCatalogEntry.MergeById( - currentManifest.SchemaCatalog - .Concat(_durableCatalog) - .Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())), - preferLast: true) - .ToArray(); - var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal); - var targetCatalog = catalogCandidates - .Where(entry => usedSchemaIds.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(usedSchemaIds.Contains)) - .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + // A record this cache holds, unchanged since the manifest was read, is rewritten under its registered schema's id, as a dirty + // one is: the manifest entry that lists an older id for it is then not needed to publish it. + var restamped = currentIndex.Values + .Where(record => !_dirtyKeys.ContainsKey(record.Key) && + Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && + !string.Equals(known.Descriptor.SchemaId, record.SchemaId, StringComparison.Ordinal)) + .Select(record => record.Key) .ToArray(); var keysToWrite = _dirtyKeys.Keys + .Concat(restamped) .OrderBy(value => value, StringComparer.Ordinal) .ToArray(); foreach (var key in keysToWrite) @@ -285,6 +283,14 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore pagePayload); } + var targetCatalog = CultSchemaCatalogEntry.Derive( + currentIndex.Values, + Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), + currentManifest.SchemaCatalog.Concat(_durableCatalog).ToArray()) + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) + .ToArray(); + // The directory store writes only its dirty pages, so it keeps a manifest already marked. var header = HeaderFor( keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds), diff --git a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md index 4fa024fe..ece39387 100644 --- a/src/GameCult.Caching/Contracts/cultcache-persistence-format.md +++ b/src/GameCult.Caching/Contracts/cultcache-persistence-format.md @@ -128,9 +128,20 @@ record with its registered schema's id, whatever id the record arrived under. A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and only when no entry owns the id to an entry that lists it as a compatible id (the first such entry -in catalog order). Two entries that share a `schemaId` are one schema seen by different writers: a -writer that merges them writes one entry whose compatible ids are the union of theirs, whatever -order they arrive in, so it lists every id any record it writes carries. +in catalog order). + +A writer derives the catalog from the records it writes, and from nothing else. For each distinct +schema id a record carries it chooses exactly one entry that publishes it: the entry that owns the id +(a registered schema's descriptor, else an entry the record arrived with), else an entry that lists +it as compatible (again registered first). When a registered descriptor and an arrived entry share an +own id, the descriptor wins: a rename or a stale content hash takes the descriptor's name, hash, +members and compatible ids as they are. Entries are written as chosen, deduplicated by own id, with +no union of compatible ids and no dependence on the order records arrive in; an entry no record needs +is not written. A write is refused, and the store left as it was, when two entries of one tier share +an own id and disagree on the schema name, when records of different types share a schema id, or when +a record's id is published by no chosen entry. Each runtime refuses with one typed error naming the +id, the schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError` +(Rust, TypeScript, Python). This is not decorative paperwork. It is what allows another CultCache implementation to inspect an old store, resolve the schema for each record, and diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 28884e10..18f5e19d 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -29,34 +29,92 @@ namespace GameCult.Caching public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty(); - // One entry per schema id, whatever order the candidates come in. Entries that share an id and a schema name are one schema seen - // by different writers: the merged entry takes its name, content hash and members from the entry being written now (the - // last with preferLast, else the first) and lists every id any of them lists. An entry of another schema under the same id is - // dropped, never merged: its ids are not this schema's. - internal static IEnumerable MergeById(IEnumerable entries, bool preferLast = false) + // The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one + // entry that publishes it: the entry that owns the id (a registered descriptor, else one that arrived with the records), else + // one that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered descriptor + // wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema names, or a + // record no chosen entry publishes, refuse the write. + // Entries that tie are taken in one fixed order, so the catalog does not depend on the order they arrive in. + private static IEnumerable Canonical(IEnumerable entries) => entries + .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.ContentHash, StringComparer.Ordinal) + .ThenBy(entry => string.Join(",", entry.CompatibleSchemaIds), StringComparer.Ordinal); + + internal static CultSchemaCatalogEntry[] Derive( + IEnumerable records, + IReadOnlyCollection registered, + IReadOnlyCollection arrived) { - foreach (var group in entries.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)) + var chosen = new Dictionary(StringComparer.Ordinal); + var carried = records.GroupBy(record => record.SchemaId, StringComparer.Ordinal).OrderBy(group => group.Key, StringComparer.Ordinal).ToArray(); + foreach (var group in carried) { - var members = group.ToArray(); - var chosen = preferLast ? members[^1] : members[0]; - var sameSchema = members.Where(entry => string.Equals(entry.SchemaName, chosen.SchemaName, StringComparison.Ordinal)).ToArray(); - if (sameSchema.Length == 1) + var schemaId = group.Key; + var recordKey = group.Min(record => record.Key)!; + var ownRegisteredAll = Canonical(registered.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray(); + var ownRegistered = ownRegisteredAll.FirstOrDefault(); + var ownArrived = Canonical(arrived.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray(); + // Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones. + var disagreeing = ownRegisteredAll.Length > 0 ? ownRegisteredAll : ownArrived; + if (disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).Count() > 1) { - yield return chosen; - continue; + var names = disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).ToArray(); + throw new CultSchemaConflictException( + $"Schema id '{schemaId}' is claimed by schemas {string.Join(" and ", names.Select(name => $"'{name}'"))}; record '{recordKey}' cannot be written under it.", + schemaId, names, recordKey); } - yield return new CultSchemaCatalogEntry + CultSchemaCatalogEntry? pick; + bool isRegistered; + if (ownRegistered != null) { - SchemaId = chosen.SchemaId, - SchemaName = chosen.SchemaName, - SchemaVersion = chosen.SchemaVersion, - ContentHash = chosen.ContentHash, - CanonicalSchemaJson = chosen.CanonicalSchemaJson, - CompatibleSchemaIds = sameSchema.SelectMany(entry => entry.CompatibleSchemaIds).Distinct(StringComparer.Ordinal).ToArray(), - Members = chosen.Members - }; + pick = ownRegistered; + isRegistered = true; + } + else if (ownArrived.Length > 0) + { + pick = ownArrived[0]; + isRegistered = false; + } + else + { + pick = Canonical(registered.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + isRegistered = pick != null; + pick ??= Canonical(arrived.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault(); + } + + if (pick == null) + continue; + if (chosen.TryGetValue(pick.SchemaId, out var prior)) + { + if (prior.Registered && !isRegistered) + continue; + + // Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other. + if (prior.Registered == isRegistered && !string.Equals(prior.Entry.SchemaName, pick.SchemaName, StringComparison.Ordinal)) + { + var names = new[] { prior.Entry.SchemaName, pick.SchemaName }; + throw new CultSchemaConflictException( + $"Schema id '{pick.SchemaId}' is claimed by schemas '{names[0]}' and '{names[1]}'; record '{recordKey}' cannot be written under it.", + pick.SchemaId, names, recordKey); + } + } + chosen[pick.SchemaId] = (pick, isRegistered); } + + foreach (var group in carried) + { + if (!chosen.Values.Any(pair => string.Equals(pair.Entry.SchemaId, group.Key, StringComparison.Ordinal) || + pair.Entry.CompatibleSchemaIds.Contains(group.Key, StringComparer.Ordinal))) + { + var recordKey = group.Min(record => record.Key)!; + throw new CultSchemaConflictException( + $"No schema entry publishes schema id '{group.Key}' of record '{recordKey}'.", + group.Key, chosen.Values.Select(pair => pair.Entry.SchemaName).ToArray(), recordKey); + } + } + + return chosen.Values.Select(pair => pair.Entry).ToArray(); } } @@ -3536,7 +3594,8 @@ namespace GameCult.Caching ReadSnapshot(); WriteSnapshot( Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)), - Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()), + Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(), + Array.Empty(), Entries.Values.Any(entry => entry.HoldsIds), existingHeader: null, wholeStore: true); @@ -3569,9 +3628,6 @@ namespace GameCult.Caching var records = ontoDisk ? disk.Records.ToDictionary(record => record.Key, StringComparer.Ordinal) : Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)).ToDictionary(record => record.Key, StringComparer.Ordinal); - var catalog = ontoDisk - ? disk.SchemaCatalog - : Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()); // The committer judged the batch against its own view; the file may have moved since. A merge is judged on the // set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving), // what they removed (departing), and this batch. @@ -3581,6 +3637,20 @@ namespace GameCult.Caching records.Remove(entry.Key.Value); foreach (var entry in request.Upserts) records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload); + if (ontoDisk) + { + // A record this cache holds, unchanged since the file was read, is written under its registered schema's id, as a + // whole-view write writes it: the file's entry that lists an older id for it is then not needed to publish it. + foreach (var record in disk.Records) + { + if (records.TryGetValue(record.Key, out var carried) && ReferenceEquals(carried, record) && + Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt && + !string.Equals(known.Descriptor.SchemaId, record.SchemaId, StringComparison.Ordinal)) + { + records[record.Key] = ToPersistedRecord(known, SerializePayload); + } + } + } // Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An // unconditional commit writes this store's whole view, so what its records hold decides. A record this commit @@ -3588,7 +3658,10 @@ namespace GameCult.Caching var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal); var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) || (!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value))); - WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); + var registered = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()) + .Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())) + .ToArray(); + WriteSnapshot(records.Values, registered, ontoDisk ? disk.SchemaCatalog : Array.Empty(), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk); if (!ontoDisk) Cache?.IdsPersisted(records.Keys); foreach (var entry in request.Deletes) @@ -3660,17 +3733,15 @@ namespace GameCult.Caching } } - private void WriteSnapshot(IEnumerable records, IEnumerable catalog, bool holdsElementIds, string? existingHeader, bool wholeStore) + private void WriteSnapshot(IEnumerable records, IReadOnlyCollection registered, IReadOnlyCollection arrived, bool holdsElementIds, string? existingHeader, bool wholeStore) { var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(); - var used = new HashSet(ordered.Select(record => record.SchemaId), StringComparer.Ordinal); var snapshot = new CultPersistedStoreSnapshot { FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)), - SchemaCatalog = CultSchemaCatalogEntry.MergeById(catalog, preferLast: true) - // An entry stays while it publishes a schema id some record carries, as its own id or a compatible one. - .Where(entry => used.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(used.Contains)) + SchemaCatalog = CultSchemaCatalogEntry.Derive(ordered, registered, arrived) .OrderBy(entry => entry.SchemaName, StringComparer.Ordinal) + .ThenBy(entry => entry.SchemaId, StringComparer.Ordinal) .ToArray(), Records = ordered }; diff --git a/src/GameCult.Caching/CultSchemaConflictException.cs b/src/GameCult.Caching/CultSchemaConflictException.cs new file mode 100644 index 00000000..9419e2c5 --- /dev/null +++ b/src/GameCult.Caching/CultSchemaConflictException.cs @@ -0,0 +1,24 @@ +using System; + +namespace GameCult.Caching +{ + /// + /// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the + /// records, neither registered), or a record whose schema id no chosen entry publishes. Nothing is written, so the store stays as + /// it was. It names the id, the schema names involved and a record key. + /// + public sealed class CultSchemaConflictException : InvalidOperationException + { + public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey) + : base(message) + { + SchemaId = schemaId; + SchemaNames = schemaNames; + RecordKey = recordKey; + } + + public string SchemaId { get; } + public string[] SchemaNames { get; } + public string RecordKey { get; } + } +}