Writers derive the catalog from the records they write; the merging is deleted

Every writer (C# single-file and directory, TypeScript, Python; Rust already did) chooses, for each schema id a record carries, one
entry that publishes it: the entry that owns the id (a registered descriptor, else an arrived entry), else one that lists it as
compatible. A registered descriptor wins over an arrived entry with the same own id. Entries are written as chosen, with no union of
compatible ids and no dependence on arrival order; an entry no record needs is not written. Two entries of one tier that share an own id
and disagree on the schema name, records of different types under one id, or a record no chosen entry publishes refuse the write with a
typed error (CultSchemaConflictException, SchemaConflictError) naming the id, the names and a record key, and the store is left as it was.
CultSchemaCatalogEntry.MergeById and the Python and TypeScript merge paths are gone. A cache-held record unchanged since the file was read
is rewritten under its registered id (C# single-file and directory), so an entry that lists an older id is not needed to publish it.
Changelogs for cultnet-py (removed export) and cultcache-py.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-30 10:03:34 +02:00
commit e77246b657
14 changed files with 678 additions and 269 deletions

View file

@ -0,0 +1,8 @@
# Changelog
## Unreleased
- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a
write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload.
- The single-file writer derives the catalog from the records it writes: one entry per carried schema id, the registered entry over an
arrived one, written as chosen and never merged. A cache stamps a record with its registered schema's id.

View file

@ -9,7 +9,7 @@ from .documents import (
define_document_registry,
define_document_type,
)
from .stores import JsonLinesBackingStore, SingleFileMessagePackBackingStore, StoreUnreadableError
from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError
__all__ = [
"BackingStore",
@ -21,6 +21,7 @@ __all__ = [
"DatabaseEntryField",
"DocumentDefinition",
"JsonLinesBackingStore",
"SchemaConflictError",
"SingleFileMessagePackBackingStore",
"StoreUnreadableError",
"database_entry_field",

View file

@ -4,7 +4,6 @@ import base64
import json
import os
import threading
from dataclasses import replace
from pathlib import Path
from typing import Any
@ -34,6 +33,20 @@ class StoreUnreadableError(ValueError):
self.path = path
class SchemaConflictError(ValueError):
"""A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id
and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names
involved (`schema_names`) and a record key (`record_key`)."""
def __init__(self, schema_id: str, schema_names: list[str], record_key: str) -> None:
super().__init__(
f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}"
)
self.schema_id = schema_id
self.schema_names = schema_names
self.record_key = record_key
class JsonLinesBackingStore:
def __init__(self, path: str | os.PathLike[str]) -> None:
self.path = Path(path)
@ -125,27 +138,21 @@ class SingleFileMessagePackBackingStore:
def push(self, envelope: CultCacheEnvelope) -> None:
with self._lock:
existing = {(item.type, item.key): item for item in self.pull_all()}
existing[(envelope.type, envelope.key)] = envelope
self._replace_all(list(existing.values()))
self._replace_all([envelope], self.pull_all())
def push_all(self, envelopes: list[CultCacheEnvelope]) -> None:
with self._lock:
existing = {(item.type, item.key): item for item in self.pull_all()}
for envelope in envelopes:
existing[(envelope.type, envelope.key)] = envelope
self._replace_all(list(existing.values()))
self._replace_all(list(envelopes), self.pull_all())
def delete(self, type: str, key: str) -> None:
with self._lock:
existing = [item for item in self.pull_all() if not (item.type == type and item.key == key)]
self._replace_all(existing)
self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)])
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None:
msgpack = self._msgpack()
self.path.parent.mkdir(parents=True, exist_ok=True)
temp = self.path.with_suffix(self.path.suffix + ".tmp")
temp.write_bytes(msgpack.packb(_encode_snapshot(envelopes, self._format), use_bin_type=True))
temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True))
temp.replace(self.path)
@staticmethod
@ -160,46 +167,81 @@ class SingleFileMessagePackBackingStore:
return msgpack
def _encode_snapshot(envelopes: list[CultCacheEnvelope], format_version: str) -> list[Any]:
# Every record's schema id is published by a catalog entry: the entry the envelope carries when it publishes that id (as its
# own id or a compatible one), else a default entry under the record's id. The catalog is keyed by the entry's own id, so a
# record read under a compatible id is written back beside the entry that lists it.
catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {}
def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]:
"""Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in."""
return sorted(candidates, key=lambda entry: (entry.schema_name, entry.content_hash, ",".join(entry.compatible_schema_ids)))
def place(envelope: CultCacheEnvelope, candidates: list[CultCacheSchemaCatalogEntry]) -> None:
for candidate in candidates:
existing = catalog_by_schema_id.get(candidate.schema_id)
if existing is None:
catalog_by_schema_id[candidate.schema_id] = candidate
return
# Entries that share an id and a schema name are one schema seen by different writers: the entry written lists every id
# any of them lists. An entry of another schema is never merged: the record gets a default entry under its own id.
if existing.schema_name == candidate.schema_name:
catalog_by_schema_id[candidate.schema_id] = replace(
existing,
compatible_schema_ids=tuple(dict.fromkeys((*existing.compatible_schema_ids, *candidate.compatible_schema_ids))),
)
return
raise ValueError(f"CultCache schema id {_schema_id_for(envelope)!r} cannot identify two different schemas")
# A record's own id is claimed first, so an entry that lists it as compatible cannot take it from the schema that owns it,
# whatever order the records arrive in.
listed: list[tuple[CultCacheEnvelope, CultCacheSchemaCatalogEntry]] = []
for envelope in envelopes:
def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultCacheSchemaCatalogEntry]:
"""The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries,
one entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one
that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an
arrived one with the same own id. Records of different types under one id, two arrived entries with the same own id and
different schema names, or a record no chosen entry publishes, refuse the write. `records` are (envelope, registered): an
envelope the caller supplied is registered, one read back from the file is arrived."""
entries: list[tuple[CultCacheSchemaCatalogEntry, bool]] = []
by_id: dict[str, list[CultCacheEnvelope]] = {}
for envelope, registered in records:
schema_id = _schema_id_for(envelope)
supplied = envelope.catalog_entry
if supplied is not None and supplied.schema_id != schema_id and schema_id in supplied.compatible_schema_ids:
listed.append((envelope, supplied))
elif supplied is not None and supplied.schema_id == schema_id:
place(envelope, [supplied])
else:
place(envelope, [_default_catalog_entry(envelope)])
for envelope, supplied in listed:
place(envelope, [supplied, _default_catalog_entry(envelope)])
publishes = supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids)
entries.append((supplied if publishes else _default_catalog_entry(envelope), registered))
by_id.setdefault(schema_id, []).append(envelope)
chosen: dict[str, tuple[CultCacheSchemaCatalogEntry, bool]] = {}
for schema_id in sorted(by_id):
carrying = by_id[schema_id]
record_key = min(envelope.key for envelope in carrying)
types = sorted({envelope.type for envelope in carrying})
if len(types) > 1:
raise SchemaConflictError(schema_id, types, record_key)
own_registered = next(iter(canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)), None)
own_arrived = canonical(entry for entry, registered in entries if not registered and entry.schema_id == schema_id)
own_registered_all = canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)
# Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones.
names = sorted({entry.schema_name for entry in (own_registered_all or own_arrived)})
if len(names) > 1:
raise SchemaConflictError(schema_id, names, record_key)
if own_registered is not None:
pick, registered_pick = own_registered, True
elif own_arrived:
pick, registered_pick = own_arrived[0], False
else:
listed_registered = next(iter(canonical(entry for entry, registered in entries if registered and schema_id in entry.compatible_schema_ids)), None)
listed_arrived = next(iter(canonical(entry for entry, registered in entries if not registered and schema_id in entry.compatible_schema_ids)), None)
pick = listed_registered if listed_registered is not None else listed_arrived
registered_pick = listed_registered is not None
if pick is None:
continue
prior = chosen.get(pick.schema_id)
if prior is not None:
if prior[1] and not registered_pick:
continue
# Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other.
if prior[1] == registered_pick and prior[0].schema_name != pick.schema_name:
raise SchemaConflictError(pick.schema_id, sorted({prior[0].schema_name, pick.schema_name}), record_key)
chosen[pick.schema_id] = (pick, registered_pick)
for schema_id in sorted(by_id):
if not any(entry.schema_id == schema_id or schema_id in entry.compatible_schema_ids for entry, _ in chosen.values()):
raise SchemaConflictError(
schema_id, sorted(entry.schema_name for entry, _ in chosen.values()), min(envelope.key for envelope in by_id[schema_id])
)
return [entry for entry, _ in chosen.values()]
def _encode_snapshot(
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
) -> list[Any]:
replaced = {(envelope.type, envelope.key) for envelope in supplied}
kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced]
envelopes = [*kept, *supplied]
catalog = [
_encode_catalog_entry(entry)
for entry in sorted(catalog_by_schema_id.values(), key=lambda item: item.schema_name)
for entry in sorted(
_derive_catalog([*((envelope, False) for envelope in kept), *((envelope, True) for envelope in supplied)]),
key=lambda item: (item.schema_name, item.schema_id),
)
]
records = [
[envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]

View file

@ -315,68 +315,130 @@ class CultCacheTests(unittest.TestCase):
self.assertEqual([entry[0] for entry in stored[1]], ["tests.store-publishes.record"])
self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 1)
# Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists,
# whichever order the records arrive in.
def test_the_store_writer_merges_the_compatible_ids_of_entries_that_share_a_schema_id_in_either_order(self) -> None:
import msgpack # type: ignore
# The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever
# order the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id.
@staticmethod
def _writer_entry(schema_id: str, name: str, content_hash: str, compatible: tuple[str, ...]):
from cultcache_py import CultCacheSchemaCatalogEntry
from cultcache_py.stores import _encode_snapshot
def entry(compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry:
return CultCacheSchemaCatalogEntry(
schema_id="tests.merge.x", schema_name="tests.merge", schema_version="tests.merge.v1",
content_hash="tests.merge.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(),
)
return CultCacheSchemaCatalogEntry(
schema_id=schema_id, schema_name=name, schema_version=name + ".v1", content_hash=content_hash,
canonical_schema_json="", compatible_schema_ids=compatible, members=(),
)
def record(key: str, schema_id: str, compatible: tuple[str, ...]) -> CultCacheEnvelope:
return CultCacheEnvelope(
key=key, type="tests.merge", payload=msgpack.packb({"key": key}, use_bin_type=True),
stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=entry(compatible),
)
def _writer_record(self, key: str, type: str, schema_id: str, catalog_entry=None) -> CultCacheEnvelope:
import msgpack # type: ignore
a = record("a", "tests.merge.x", ("tests.merge.x",))
b = record("b", "tests.merge.y", ("tests.merge.x", "tests.merge.y"))
for order in ([a, b], [b, a]):
catalog = _encode_snapshot(order, "cultcache.store.v1")[1]
self.assertEqual([sorted(entry[5]) for entry in catalog], [["tests.merge.x", "tests.merge.y"]])
return CultCacheEnvelope(
key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True),
stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry,
)
@staticmethod
def _raw_entry(entry) -> list:
return [entry.schema_id, entry.schema_name, entry.schema_version, entry.content_hash, "", list(entry.compatible_schema_ids), []]
def test_a_registered_entry_is_written_as_it_is_over_the_arrived_entries_with_the_same_own_id_in_either_order(self) -> None:
import msgpack # type: ignore
stale1 = self._writer_entry("tests.x", "tests.n", "stale-1", ("tests.x", "old"))
stale2 = self._writer_entry("tests.x", "tests.n", "stale-2", ("tests.x",))
registered = self._writer_record("a", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "fresh", ("tests.x",)))
for entries in ([stale1, stale2], [stale2, stale1]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(entry) for entry in entries],
[["b", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"b": 1}, use_bin_type=True)]],
], use_bin_type=True))
SingleFileMessagePackBackingStore(path).push(registered)
catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1]
self.assertEqual(len(catalog), 1)
self.assertEqual(catalog[0][3], "fresh")
self.assertEqual(catalog[0][5], ["tests.x"], "written as chosen, no union with the arrived entries' ids")
def test_a_rename_with_a_stable_schema_id_a_whole_view_write_names_the_registered_schema(self) -> None:
import msgpack # type: ignore
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
SingleFileMessagePackBackingStore(path).push_all([b])
SingleFileMessagePackBackingStore(path).push(a)
self.assertEqual(len(SingleFileMessagePackBackingStore(path).pull_all()), 2)
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.old", "stale", ("tests.x",)))],
[["a", "tests.x", "2026-09-30T00:00:00Z", msgpack.packb({"a": 1}, use_bin_type=True)]],
], use_bin_type=True))
SingleFileMessagePackBackingStore(path).push_all(
[self._writer_record("a", "tests.new", "tests.x", self._writer_entry("tests.x", "tests.new", "fresh", ("tests.x",)))]
)
catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1]
self.assertEqual([(entry[1], entry[3]) for entry in catalog], [("tests.new", "fresh")])
def test_two_entries_of_one_tier_that_share_an_own_id_and_disagree_on_the_schema_name_refuse_the_write_typed(self) -> None:
from cultcache_py import SchemaConflictError
first = self._writer_record("b", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.first", "h1", ("tests.x",)))
second = self._writer_record("a", "tests.first", "tests.x", self._writer_entry("tests.x", "tests.second", "h2", ("tests.x",)))
for order in ([first, second], [second, first]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(path).push_all(order)
self.assertEqual(refused.exception.schema_id, "tests.x")
self.assertEqual(refused.exception.record_key, "a")
self.assertEqual(refused.exception.schema_names, ["tests.first", "tests.second"])
self.assertFalse(path.exists())
def test_records_of_different_types_under_one_schema_id_refuse_the_write_typed_and_no_record_changes_type(self) -> None:
from cultcache_py import SchemaConflictError
a = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.x", "tests.n", "h1", ("tests.x", "tests.y")))
b = self._writer_record("b", "tests.m", "tests.y", self._writer_entry("tests.y", "tests.m", "h2", ("tests.y",)))
for order in ([a, b], [b, a]):
with tempfile.TemporaryDirectory() as tmp:
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(Path(tmp) / "store.msgpack").push_all(order)
self.assertEqual(refused.exception.schema_id, "tests.y")
self.assertEqual(refused.exception.schema_names, ["tests.m", "tests.n"])
def test_an_entry_that_owns_an_id_is_chosen_over_one_that_lists_it(self) -> None:
import msgpack # type: ignore
owner = self._writer_record("a", "tests.n", "tests.y", self._writer_entry("tests.y", "tests.n", "h1", ("tests.y",)))
lister = self._writer_record("b", "tests.n", "tests.x", self._writer_entry("tests.x", "tests.n", "h2", ("tests.x", "tests.y")))
for order in ([lister, owner], [owner, lister]):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
SingleFileMessagePackBackingStore(path).push_all(order)
catalog = msgpack.unpackb(path.read_bytes(), raw=False)[1]
self.assertEqual(sorted((entry[0], entry[5]) for entry in catalog), [("tests.x", ["tests.x", "tests.y"]), ("tests.y", ["tests.y"])])
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
SingleFileMessagePackBackingStore(path).push_all([owner])
self.assertEqual([entry[0] for entry in msgpack.unpackb(path.read_bytes(), raw=False)[1]], ["tests.y"])
def test_a_record_no_chosen_entry_publishes_refuses_the_write_typed_and_leaves_the_file(self) -> None:
import msgpack # type: ignore
from cultcache_py import SchemaConflictError
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "store.msgpack"
path.write_bytes(msgpack.packb([
"cultcache.store.v1", [self._raw_entry(self._writer_entry("tests.x", "tests.legacy", "stale", ("tests.x", "old")))],
[["z", "old", "2026-09-30T00:00:00Z", msgpack.packb({"z": 1}, use_bin_type=True)]],
], use_bin_type=True))
before = path.read_bytes()
with self.assertRaises(SchemaConflictError) as refused:
SingleFileMessagePackBackingStore(path).push(
self._writer_record("a", "tests.legacy", "tests.x", self._writer_entry("tests.x", "tests.legacy", "fresh", ("tests.x",)))
)
self.assertEqual(refused.exception.schema_id, "old")
self.assertEqual(refused.exception.record_key, "z")
self.assertEqual(path.read_bytes(), before)
def test_an_id_an_entry_owns_names_that_entry_not_one_that_lists_it_as_compatible(self) -> None:
for vector in ("own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"):
store = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "readability" / vector)
self.assertEqual([envelope.type for envelope in store.pull_all()], ["vectors.item", "vectors.item"], vector)
# Entries of different schemas that share an id are not merged: the record gets a default entry under its own id, so no record
# is read as another schema, whichever order the records arrive in.
def test_the_store_writer_never_merges_an_entry_of_another_schema(self) -> None:
import msgpack # type: ignore
from cultcache_py import CultCacheSchemaCatalogEntry
from cultcache_py.stores import _decode_snapshot, _encode_snapshot
def entry(name: str, compatible: tuple[str, ...]) -> CultCacheSchemaCatalogEntry:
return CultCacheSchemaCatalogEntry(
schema_id="tests.retype.x", schema_name=name, schema_version=name + ".v1",
content_hash="tests.retype.x", canonical_schema_json="", compatible_schema_ids=compatible, members=(),
)
def record(key: str, type: str, schema_id: str, catalog_entry: CultCacheSchemaCatalogEntry) -> CultCacheEnvelope:
return CultCacheEnvelope(
key=key, type=type, payload=msgpack.packb({"key": key}, use_bin_type=True),
stored_at="2026-09-30T00:00:00Z", schema_id=schema_id, catalog_entry=catalog_entry,
)
a = record("a", "tests.retype.mine", "tests.retype.x", entry("tests.retype.mine", ("tests.retype.x",)))
b = record("b", "tests.retype.theirs", "tests.retype.y", entry("tests.retype.theirs", ("tests.retype.x", "tests.retype.y")))
for order in ([a, b], [b, a]):
decoded = msgpack.unpackb(msgpack.packb(_encode_snapshot(order, "cultcache.store.v1"), use_bin_type=True), raw=False)
_, envelopes = _decode_snapshot(decoded)
self.assertEqual(sorted((e.key, e.type) for e in envelopes), [("a", "tests.retype.mine"), ("b", "tests.retype.theirs")])
def test_a_record_loaded_under_a_compatible_schema_id_is_written_back_under_the_id_its_catalog_entry_carries(self) -> None:
import msgpack # type: ignore

View file

@ -389,6 +389,27 @@ impl std::error::Error for StoreUnreadableError {
}
}
/// A write the catalog cannot describe: records of different types under one schema id. This writer derives one entry per carried
/// schema id from its records, so that is its only conflict. Nothing is written. It names the id, the two names and a record key.
#[derive(Debug)]
pub struct SchemaConflictError {
pub schema_id: String,
pub schema_names: Vec<String>,
pub record_key: String,
}
impl std::fmt::Display for SchemaConflictError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
formatter,
"schema id {:?} cannot describe record {:?}: {:?}",
self.schema_id, self.record_key, self.schema_names
)
}
}
impl std::error::Error for SchemaConflictError {}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct PushAllOptions {
pub soft: bool,
@ -2592,11 +2613,13 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope], format: &str) -> Result<
.clone()
.unwrap_or_else(|| entry.r#type.clone());
if let Some(existing_type) = schema_types.insert(schema_id.clone(), entry.r#type.clone()) {
ensure!(
existing_type == entry.r#type,
"CultCache schema {schema_id:?} cannot identify both {existing_type:?} and {:?}",
entry.r#type
);
if existing_type != entry.r#type {
return Err(anyhow::Error::new(SchemaConflictError {
schema_id,
schema_names: vec![existing_type, entry.r#type.clone()],
record_key: entry.key.clone(),
}));
}
}
}
@ -4842,6 +4865,33 @@ mod tests {
Ok(())
}
// Records of different types under one schema id cannot be described by one catalog entry: the write is refused, typed, in
// either order, and the file is left as it was. A rename with a stable id (same type name in the new schema) writes fine.
#[test]
fn records_of_different_types_under_one_schema_id_refuse_the_write_typed() -> Result<()> {
let record = |key: &str, r#type: &str| CultCacheEnvelope {
key: key.into(),
r#type: r#type.into(),
payload: b"one".to_vec(),
stored_at: "2026-09-30T00:00:00Z".into(),
schema_id: Some("tests.x".into()),
};
for order in [vec![record("a", "tests.n"), record("b", "tests.m")], vec![record("b", "tests.m"), record("a", "tests.n")]] {
let temp = tempfile::tempdir()?;
let path = temp.path().join("store.msgpack");
let mut store = SingleFileMessagePackBackingStore::new(&path);
let error = store.push_all(&order, PushAllOptions::default()).unwrap_err();
let conflict = error.downcast_ref::<SchemaConflictError>().unwrap_or_else(|| panic!("{error:#}"));
assert_eq!(conflict.schema_id, "tests.x");
let mut names = conflict.schema_names.clone();
names.sort();
assert_eq!(names, vec!["tests.m".to_string(), "tests.n".to_string()]);
assert!(conflict.record_key == "a" || conflict.record_key == "b");
assert!(!path.exists(), "nothing is written");
}
Ok(())
}
// A store cut short is refused whatever its version, and left as it is.
#[test]
fn a_truncated_v1_or_v2_store_is_refused_on_open_and_rewrite() -> Result<()> {

View file

@ -2,7 +2,7 @@ export { CultCache, CultCacheBuilder } from "./cult-cache";
export { inspectCultCacheBytes } from "./cult-cache-inspector";
export { defineDocumentRegistry, defineDocumentType } from "./document";
export { SingleFileMessagePackBackingStore } from "./single-file-messagepack-backing-store";
export { StoreUnreadableError } from "./store-format";
export { SchemaConflictError, StoreUnreadableError } from "./store-format";
export type {
AnyCultCacheDocumentDefinition,
CacheBackingStore,

View file

@ -4,7 +4,7 @@ import { dirname, resolve } from "node:path";
import { decode, encode } from "@msgpack/msgpack";
import { z } from "zod";
import { STORE_FORMAT_VERSION, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format";
import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, requireStoreSlots, requireV1RecordSlots } from "./store-format";
import type {
CacheBackingStore,
CultCacheEnvelope,
@ -49,7 +49,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
const disk = await this.#readDisk();
this.#format = disk.format;
if (disk.repairedLegacyPayload) {
await this.#writeAll(disk.envelopes);
await this.#writeAll([], disk.envelopes);
}
return disk.envelopes;
@ -61,8 +61,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
const filtered = existing.filter(
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
);
filtered.push(entry);
await this.#writeAll(filtered);
await this.#writeAll([entry], filtered);
});
}
@ -72,7 +71,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
const filtered = existing.filter(
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
);
await this.#writeAll(filtered);
await this.#writeAll([], filtered);
});
}
@ -95,7 +94,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
// reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused
// and left as it is.
this.#format = (await this.#readDisk()).format;
await this.#writeAll(entries);
await this.#writeAll(entries, []);
});
}
@ -140,7 +139,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
return result;
}
async #writeAll(entries: CultCacheEnvelope[]): Promise<void> {
async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise<void> {
await mkdir(dirname(this.filePath), { recursive: true });
const tempPath = `${this.filePath}.tmp-${process.pid}-${Date.now()}-${Math.random()
@ -148,7 +147,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
.slice(2)}`;
try {
await writeFile(tempPath, encode(encodeSnapshot(entries, this.#format)));
await writeFile(tempPath, encode(encodeSnapshot(supplied, arrived, this.#format)));
await renameWithRetry(tempPath, this.filePath);
} catch (error) {
await rm(tempPath, { force: true }).catch(() => undefined);
@ -196,9 +195,14 @@ type DecodedSnapshot = {
records: PersistedRecord[];
};
function encodeSnapshot(entries: CultCacheEnvelope[], format: StoreFormat): unknown[] {
const catalog = [...catalogEntriesFor(entries).values()]
.sort((left, right) => compareOrdinal(left.schemaName, right.schemaName));
function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] {
const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`));
const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`));
const entries = [...kept, ...supplied];
const catalog = catalogEntriesFor([
...kept.map((envelope) => ({ envelope, registered: false })),
...supplied.map((envelope) => ({ envelope, registered: true })),
]).sort((left, right) => compareOrdinal(left.schemaName, right.schemaName) || compareOrdinal(left.schemaId, right.schemaId));
const records = [...entries]
.sort((left, right) => compareOrdinal(left.key, right.key))
.map((entry) => [
@ -215,71 +219,113 @@ function encodeSnapshot(entries: CultCacheEnvelope[], format: StoreFormat): unkn
];
}
// Every record's schema id is published by a catalog entry: the entry the envelope carries when it publishes that id (as its
// own id or a compatible one), else a default entry under the record's id. The catalog is keyed by the entry's own id, so a
// record read under a compatible id is written back beside the entry that lists it.
function catalogEntriesFor(entries: CultCacheEnvelope[]): Map<string, CultCacheSchemaCatalogEntry> {
const catalog = new Map<string, CultCacheSchemaCatalogEntry>();
const defaultEntryFor = (entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry => {
const schemaId = schemaIdForEnvelope(entry);
return {
schemaId,
// The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one
// entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one that
// lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an arrived
// one with the same own id. Records of different types under one id, two arrived entries with the same own id and different schema
// names, or a record no chosen entry publishes, refuse the write. An envelope the caller supplied is registered; one read back
// from the file is arrived.
function catalogEntriesFor(
records: Array<{ envelope: CultCacheEnvelope; registered: boolean }>,
): CultCacheSchemaCatalogEntry[] {
const entries: Array<{ entry: CultCacheSchemaCatalogEntry; registered: boolean }> = [];
const byId = new Map<string, CultCacheEnvelope[]>();
for (const { envelope, registered } of records) {
const schemaId = schemaIdForEnvelope(envelope);
const supplied = envelope.catalogEntry;
const publishes =
supplied !== undefined && (supplied.schemaId === schemaId || (supplied.compatibleSchemaIds ?? []).includes(schemaId));
entries.push({ entry: publishes && supplied ? supplied : defaultEntryFor(envelope), registered });
byId.set(schemaId, [...(byId.get(schemaId) ?? []), envelope]);
}
const lists = (entry: CultCacheSchemaCatalogEntry, schemaId: string): boolean =>
(entry.compatibleSchemaIds ?? [entry.schemaId]).includes(schemaId);
const chosen = new Map<string, { entry: CultCacheSchemaCatalogEntry; registered: boolean }>();
for (const schemaId of [...byId.keys()].sort(compareOrdinal)) {
const carrying = byId.get(schemaId)!;
const recordKey = carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!;
const types = [...new Set(carrying.map((envelope) => envelope.type))].sort(compareOrdinal);
if (types.length > 1) {
throw new SchemaConflictError(schemaId, types, recordKey);
}
const ownRegistered = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId))[0];
const ownArrived = canonical(entries.filter((candidate) => !candidate.registered && candidate.entry.schemaId === schemaId));
// Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones.
const ownRegisteredAll = canonical(entries.filter((candidate) => candidate.registered && candidate.entry.schemaId === schemaId));
const names = [
...new Set((ownRegisteredAll.length > 0 ? ownRegisteredAll : ownArrived).map((candidate) => candidate.entry.schemaName)),
].sort(compareOrdinal);
if (names.length > 1) {
throw new SchemaConflictError(schemaId, names, recordKey);
}
let pick: { entry: CultCacheSchemaCatalogEntry; registered: boolean } | undefined;
if (ownRegistered !== undefined) {
pick = ownRegistered;
} else if (ownArrived.length > 0) {
pick = ownArrived[0];
} else {
pick =
canonical(entries.filter((candidate) => candidate.registered && lists(candidate.entry, schemaId)))[0] ??
canonical(entries.filter((candidate) => !candidate.registered && lists(candidate.entry, schemaId)))[0];
}
if (pick === undefined) {
continue;
}
const prior = chosen.get(pick.entry.schemaId);
if (prior !== undefined) {
if (prior.registered && !pick.registered) {
continue;
}
// Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other.
if (prior.registered === pick.registered && prior.entry.schemaName !== pick.entry.schemaName) {
throw new SchemaConflictError(pick.entry.schemaId, [prior.entry.schemaName, pick.entry.schemaName].sort(compareOrdinal), recordKey);
}
}
chosen.set(pick.entry.schemaId, pick);
}
for (const [schemaId, carrying] of byId) {
if (![...chosen.values()].some(({ entry }) => entry.schemaId === schemaId || lists(entry, schemaId))) {
throw new SchemaConflictError(
schemaId,
[...chosen.values()].map(({ entry }) => entry.schemaName).sort(compareOrdinal),
carrying.map((envelope) => envelope.key).sort(compareOrdinal)[0]!,
);
}
}
return [...chosen.values()].map(({ entry }) => entry);
}
// Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in.
function canonical<T extends { entry: CultCacheSchemaCatalogEntry }>(candidates: T[]): T[] {
const key = (candidate: T): string =>
[candidate.entry.schemaName, candidate.entry.contentHash ?? "", (candidate.entry.compatibleSchemaIds ?? []).join(",")].join("\u0000");
return [...candidates].sort((left, right) => compareOrdinal(key(left), key(right)));
}
function defaultEntryFor(entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry {
const schemaId = schemaIdForEnvelope(entry);
return {
schemaId,
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
contentHash: schemaId,
canonicalSchemaJson: JSON.stringify({
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
contentHash: schemaId,
canonicalSchemaJson: JSON.stringify({
schemaName: entry.type,
schemaVersion: `${entry.type}.v1`,
members: [],
}),
compatibleSchemaIds: [schemaId],
members: [],
};
}),
compatibleSchemaIds: [schemaId],
members: [],
};
const place = (entry: CultCacheEnvelope, candidates: CultCacheSchemaCatalogEntry[]): void => {
for (const candidate of candidates) {
const existing = catalog.get(candidate.schemaId);
if (existing === undefined) {
catalog.set(candidate.schemaId, candidate);
return;
}
// Entries that share an id and a schema name are one schema seen by different writers: the entry written lists every id any
// of them lists. An entry of another schema is never merged: the record gets a default entry under its own id.
if (existing.schemaName === candidate.schemaName) {
catalog.set(candidate.schemaId, {
...existing,
compatibleSchemaIds: [
...new Set([...(existing.compatibleSchemaIds ?? [existing.schemaId]), ...(candidate.compatibleSchemaIds ?? [candidate.schemaId])]),
],
});
return;
}
}
throw new Error(`CultCache schema id "${schemaIdForEnvelope(entry)}" cannot identify two different schemas.`);
};
// A record's own id is claimed first, so an entry that lists it as compatible cannot take it from the schema that owns it,
// whatever order the records arrive in.
const listed: Array<[CultCacheEnvelope, CultCacheSchemaCatalogEntry]> = [];
for (const entry of entries) {
const schemaId = schemaIdForEnvelope(entry);
const supplied = entry.catalogEntry;
if (supplied !== undefined && supplied.schemaId !== schemaId && (supplied.compatibleSchemaIds ?? []).includes(schemaId)) {
listed.push([entry, supplied]);
} else if (supplied !== undefined && supplied.schemaId === schemaId) {
place(entry, [supplied]);
} else {
place(entry, [defaultEntryFor(entry)]);
}
}
for (const [entry, supplied] of listed) {
place(entry, [supplied, defaultEntryFor(entry)]);
}
return catalog;
}
function schemaIdForEnvelope(entry: CultCacheEnvelope): string {

View file

@ -32,6 +32,25 @@ export function isStoreSnapshot(decoded: unknown): decoded is [StoreFormat, ...u
return true;
}
/**
* A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id and
* disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names involved and
* a record key.
*/
export class SchemaConflictError extends Error {
readonly schemaId: string;
readonly schemaNames: string[];
readonly recordKey: string;
constructor(schemaId: string, schemaNames: string[], recordKey: string) {
super(`Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`);
this.name = "SchemaConflictError";
this.schemaId = schemaId;
this.schemaNames = schemaNames;
this.recordKey = recordKey;
}
}
/**
* A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra slot),
* a header or record it does not know, or a body it cannot decode. Open, flush and every rewrite refuse a file with this

View file

@ -15,7 +15,7 @@ import { CultCache } from "../src/cult-cache";
import { inspectCultCacheBytes } from "../src/cult-cache-inspector";
import { defineDocumentRegistry, defineDocumentType } from "../src/document";
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
import { StoreUnreadableError } from "../src/store-format";
import { SchemaConflictError, StoreUnreadableError } from "../src/store-format";
import type { CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types";
const execFileAsync = promisify(execFile);
@ -1344,78 +1344,138 @@ test("the store publishes a record's schema id even when the envelope's catalog
assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 1);
});
// Entries that share a schema id are one schema seen by different writers: the catalog written lists every id any of them lists,
// whichever order the records arrive in.
test("the store writer merges the compatible ids of entries that share a schema id, in either order", async () => {
const entry = (compatibleSchemaIds: string[]) => ({
schemaId: "tests.merge.x",
schemaName: "tests.merge",
schemaVersion: "tests.merge.v1",
contentHash: "tests.merge.x",
canonicalSchemaJson: "",
compatibleSchemaIds,
members: [],
});
const record = (key: string, schemaId: string, compatibleSchemaIds: string[]) => ({
key,
type: "tests.merge",
schemaId,
payload: encode({ key }),
storedAt: "2026-09-30T00:00:00.0000000Z",
catalogEntry: entry(compatibleSchemaIds),
});
const a = record("a", "tests.merge.x", ["tests.merge.x"]);
const b = record("b", "tests.merge.y", ["tests.merge.x", "tests.merge.y"]);
const dir = await mkdtemp(join(tmpdir(), "cultcache-merge-"));
// The catalog a write leaves is derived from its records: one entry per carried id, chosen and never merged, the same whatever order
// the records arrive in. A registered (supplied) entry wins over an arrived (read back) entry with the same own id.
const writerEntry = (schemaId: string, schemaName: string, contentHash: string, compatibleSchemaIds: string[]) => ({
schemaId,
schemaName,
schemaVersion: `${schemaName}.v1`,
contentHash,
canonicalSchemaJson: "",
compatibleSchemaIds,
members: [],
});
const writerRecord = (key: string, type: string, schemaId: string, catalogEntry?: ReturnType<typeof writerEntry>) => ({
key,
type,
schemaId,
payload: encode({ key }),
storedAt: "2026-09-30T00:00:00.0000000Z",
...(catalogEntry ? { catalogEntry } : {}),
});
const rawCatalogEntry = (entry: ReturnType<typeof writerEntry>) => [
entry.schemaId,
entry.schemaName,
entry.schemaVersion,
entry.contentHash,
entry.canonicalSchemaJson,
entry.compatibleSchemaIds,
[],
];
const catalogOf = async (file: string): Promise<unknown[][]> =>
(decode(await readFile(file)) as [string, unknown[][], unknown[][]])[1];
for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) {
test("a registered entry is written as it is, over the arrived entries with the same own id, in either order", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-registered-wins-"));
const stale1 = writerEntry("tests.x", "tests.n", "stale-1", ["tests.x", "old"]);
const stale2 = writerEntry("tests.x", "tests.n", "stale-2", ["tests.x"]);
const registered = writerRecord("a", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "fresh", ["tests.x"]));
for (const [name, entries] of [["12", [stale1, stale2]], ["21", [stale2, stale1]]] as const) {
const file = join(dir, `${name}.msgpack`);
await new SingleFileMessagePackBackingStore(file).pushAll([...order]);
const decoded = decode(await readFile(file)) as [string, unknown[][], unknown[][]];
assert.deepEqual(decoded[1].map((catalogEntry) => (catalogEntry[5] as string[]).slice().sort()), [["tests.merge.x", "tests.merge.y"]], name);
assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2, name);
await writeFile(
file,
encode(["cultcache.store.v1", entries.map(rawCatalogEntry), [["b", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ b: 1 })]]]),
);
await new SingleFileMessagePackBackingStore(file).push(registered);
const catalog = await catalogOf(file);
assert.equal(catalog.length, 1, name);
assert.equal(catalog[0]![3], "fresh", `${name}: the registered entry's hash is written`);
assert.deepEqual(catalog[0]![5], ["tests.x"], `${name}: written as chosen, no union with the arrived entries' ids`);
}
// A put over a key of a file whose other record is stored under the wider entry's compatible id.
const file = join(dir, "push.msgpack");
await new SingleFileMessagePackBackingStore(file).pushAll([b]);
await new SingleFileMessagePackBackingStore(file).push(a);
assert.equal((await new SingleFileMessagePackBackingStore(file).pullAll()).length, 2);
});
// Entries of different schemas that share an id are not merged: the record gets a default entry under its own id, so no record is
// read as another schema, whichever order the records arrive in.
test("the store writer never merges an entry of another schema into a record's catalog entry", async () => {
const entry = (schemaName: string, compatibleSchemaIds: string[]) => ({
schemaId: "tests.retype.x",
schemaName,
schemaVersion: `${schemaName}.v1`,
contentHash: "tests.retype.x",
canonicalSchemaJson: "",
compatibleSchemaIds,
members: [],
});
const record = (key: string, type: string, schemaId: string, catalogEntry: ReturnType<typeof entry>) => ({
key,
type,
schemaId,
payload: encode({ key }),
storedAt: "2026-09-30T00:00:00.0000000Z",
catalogEntry,
});
// a is schema "mine" under x; b is schema "theirs" under y, carrying an entry with x's id that lists y.
const a = record("a", "tests.retype.mine", "tests.retype.x", entry("tests.retype.mine", ["tests.retype.x"]));
const b = record("b", "tests.retype.theirs", "tests.retype.y", entry("tests.retype.theirs", ["tests.retype.x", "tests.retype.y"]));
test("a rename with a stable schema id: a whole-view write names the registered schema", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-rename-"));
const file = join(dir, "rename.msgpack");
await writeFile(
file,
encode(["cultcache.store.v1", [rawCatalogEntry(writerEntry("tests.x", "tests.old", "stale", ["tests.x"]))], [["a", "tests.x", "2026-09-30T00:00:00.0000000Z", encode({ a: 1 })]]]),
);
await new SingleFileMessagePackBackingStore(file).pushAll([writerRecord("a", "tests.new", "tests.x", writerEntry("tests.x", "tests.new", "fresh", ["tests.x"]))]);
assert.deepEqual((await catalogOf(file)).map((entry) => [entry[1], entry[3]]), [["tests.new", "fresh"]]);
});
test("two entries of one tier that share an own id and disagree on the schema name refuse the write, typed", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-disagree-"));
const first = writerRecord("b", "tests.first", "tests.x", writerEntry("tests.x", "tests.first", "h1", ["tests.x"]));
const second = writerRecord("a", "tests.first", "tests.x", writerEntry("tests.x", "tests.second", "h2", ["tests.x"]));
for (const [name, order] of [["fs", [first, second]], ["sf", [second, first]]] as const) {
const file = join(dir, `${name}.msgpack`);
await assert.rejects(
() => new SingleFileMessagePackBackingStore(file).pushAll([...order]),
(error) =>
error instanceof SchemaConflictError &&
error.schemaId === "tests.x" &&
error.recordKey === "a" &&
[...error.schemaNames].sort().join() === "tests.first,tests.second",
name,
);
}
});
test("records of different types under one schema id refuse the write, typed, and no record changes type", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-retype-"));
const a = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.x", "tests.n", "h1", ["tests.x", "tests.y"]));
const b = writerRecord("b", "tests.m", "tests.y", writerEntry("tests.y", "tests.m", "h2", ["tests.y"]));
for (const [name, order] of [["ab", [a, b]], ["ba", [b, a]]] as const) {
const file = join(dir, `${name}.msgpack`);
await new SingleFileMessagePackBackingStore(file).pushAll([...order]);
const back = await new SingleFileMessagePackBackingStore(file).pullAll();
assert.deepEqual(back.map((envelope) => [envelope.key, envelope.type]).sort(), [["a", "tests.retype.mine"], ["b", "tests.retype.theirs"]], name);
await assert.rejects(
() => new SingleFileMessagePackBackingStore(file).pushAll([...order]),
(error) => error instanceof SchemaConflictError && error.schemaId === "tests.y" && error.schemaNames.join() === "tests.m,tests.n",
name,
);
}
});
test("an entry that owns an id is chosen over one that lists it, and an entry no record needs is not written", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-own-first-"));
const owner = writerRecord("a", "tests.n", "tests.y", writerEntry("tests.y", "tests.n", "h1", ["tests.y"]));
const lister = writerRecord("b", "tests.n", "tests.x", writerEntry("tests.x", "tests.n", "h2", ["tests.x", "tests.y"]));
const onlyOwner = join(dir, "own.msgpack");
await new SingleFileMessagePackBackingStore(onlyOwner).pushAll([owner]);
assert.deepEqual((await catalogOf(onlyOwner)).map((entry) => entry[0]), ["tests.y"]);
// The lister carried only its own id: its list of y is not needed, and the owner's entry publishes y.
for (const [name, order] of [["lo", [lister, owner]], ["ol", [owner, lister]]] as const) {
const file = join(dir, `${name}.msgpack`);
await new SingleFileMessagePackBackingStore(file).pushAll([...order]);
const catalog = await catalogOf(file);
assert.deepEqual(catalog.map((entry) => [entry[0], entry[5]]).sort(), [["tests.x", ["tests.x", "tests.y"]], ["tests.y", ["tests.y"]]], name);
}
});
test("a record no chosen entry publishes refuses the write, typed, and leaves the file", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-orphan-"));
const file = join(dir, "orphan.msgpack");
// z sits under old, which only the arrived entry (id x) lists; the supplied entry owns x and does not.
await writeFile(
file,
encode([
"cultcache.store.v1",
[rawCatalogEntry(writerEntry("tests.x", "tests.legacy", "stale", ["tests.x", "old"]))],
[["z", "old", "2026-09-30T00:00:00.0000000Z", encode({ z: 1 })]],
]),
);
const before = await readFile(file);
await assert.rejects(
() => new SingleFileMessagePackBackingStore(file).push(writerRecord("a", "tests.legacy", "tests.x", writerEntry("tests.x", "tests.legacy", "fresh", ["tests.x"]))),
(error) => error instanceof SchemaConflictError && error.schemaId === "old" && error.recordKey === "z",
);
assert.ok(before.equals(await readFile(file)), "the file is left as it was");
});
test("an id an entry owns names that entry, not one that lists it as compatible", async () => {
const dir = await mkdtemp(join(tmpdir(), "cultcache-own-id-"));
for (const vector of ["own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"]) {

View file

@ -0,0 +1,9 @@
# Changelog
## Unreleased (breaking; version 0.x, so the next minor)
- Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one
its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`,
which returns the document; take the schema id from `document.catalog_entry().schema_id`.
- `apply_raw_document_record`, `apply_raw_snapshot` and `apply_shard_log_response` report and stamp the local schema id, also for a record
that arrived under a compatible id or the schema name.

View file

@ -251,18 +251,16 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
currentIndex[key] = ToIndexRecord(stored);
}
var catalogCandidates = CultSchemaCatalogEntry.MergeById(
currentManifest.SchemaCatalog
.Concat(_durableCatalog)
.Concat(Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())),
preferLast: true)
.ToArray();
var usedSchemaIds = currentIndex.Values.Select(record => record.SchemaId).ToHashSet(StringComparer.Ordinal);
var targetCatalog = catalogCandidates
.Where(entry => usedSchemaIds.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(usedSchemaIds.Contains))
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
// A record this cache holds, unchanged since the manifest was read, is rewritten under its registered schema's id, as a dirty
// one is: the manifest entry that lists an older id for it is then not needed to publish it.
var restamped = currentIndex.Values
.Where(record => !_dirtyKeys.ContainsKey(record.Key) &&
Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt &&
!string.Equals(known.Descriptor.SchemaId, record.SchemaId, StringComparison.Ordinal))
.Select(record => record.Key)
.ToArray();
var keysToWrite = _dirtyKeys.Keys
.Concat(restamped)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
foreach (var key in keysToWrite)
@ -285,6 +283,14 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
pagePayload);
}
var targetCatalog = CultSchemaCatalogEntry.Derive(
currentIndex.Values,
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(),
currentManifest.SchemaCatalog.Concat(_durableCatalog).ToArray())
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ThenBy(entry => entry.SchemaId, StringComparer.Ordinal)
.ToArray();
// The directory store writes only its dirty pages, so it keeps a manifest already marked.
var header = HeaderFor(
keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds),

View file

@ -128,9 +128,20 @@ record with its registered schema's id, whatever id the record arrived under.
A reader resolves a record's schema id to the entry that owns it, that entry's own `schemaId`, and
only when no entry owns the id to an entry that lists it as a compatible id (the first such entry
in catalog order). Two entries that share a `schemaId` are one schema seen by different writers: a
writer that merges them writes one entry whose compatible ids are the union of theirs, whatever
order they arrive in, so it lists every id any record it writes carries.
in catalog order).
A writer derives the catalog from the records it writes, and from nothing else. For each distinct
schema id a record carries it chooses exactly one entry that publishes it: the entry that owns the id
(a registered schema's descriptor, else an entry the record arrived with), else an entry that lists
it as compatible (again registered first). When a registered descriptor and an arrived entry share an
own id, the descriptor wins: a rename or a stale content hash takes the descriptor's name, hash,
members and compatible ids as they are. Entries are written as chosen, deduplicated by own id, with
no union of compatible ids and no dependence on the order records arrive in; an entry no record needs
is not written. A write is refused, and the store left as it was, when two entries of one tier share
an own id and disagree on the schema name, when records of different types share a schema id, or when
a record's id is published by no chosen entry. Each runtime refuses with one typed error naming the
id, the schema names and a record key: `CultSchemaConflictException` (C#), `SchemaConflictError`
(Rust, TypeScript, Python).
This is not decorative paperwork. It is what allows another CultCache
implementation to inspect an old store, resolve the schema for each record, and

View file

@ -29,34 +29,92 @@ namespace GameCult.Caching
public CultSchemaMemberCatalogEntry[] Members { get; set; } = Array.Empty<CultSchemaMemberCatalogEntry>();
// One entry per schema id, whatever order the candidates come in. Entries that share an id and a schema name are one schema seen
// by different writers: the merged entry takes its name, content hash and members from the entry being written now (the
// last with preferLast, else the first) and lists every id any of them lists. An entry of another schema under the same id is
// dropped, never merged: its ids are not this schema's.
internal static IEnumerable<CultSchemaCatalogEntry> MergeById(IEnumerable<CultSchemaCatalogEntry> entries, bool preferLast = false)
// The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries, one
// entry that publishes it: the entry that owns the id (a registered descriptor, else one that arrived with the records), else
// one that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered descriptor
// wins over an arrived entry with the same own id. Two arrived entries with the same own id and different schema names, or a
// record no chosen entry publishes, refuse the write.
// Entries that tie are taken in one fixed order, so the catalog does not depend on the order they arrive in.
private static IEnumerable<CultSchemaCatalogEntry> Canonical(IEnumerable<CultSchemaCatalogEntry> entries) => entries
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ThenBy(entry => entry.ContentHash, StringComparer.Ordinal)
.ThenBy(entry => string.Join(",", entry.CompatibleSchemaIds), StringComparer.Ordinal);
internal static CultSchemaCatalogEntry[] Derive(
IEnumerable<CultPersistedRecord> records,
IReadOnlyCollection<CultSchemaCatalogEntry> registered,
IReadOnlyCollection<CultSchemaCatalogEntry> arrived)
{
foreach (var group in entries.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal))
var chosen = new Dictionary<string, (CultSchemaCatalogEntry Entry, bool Registered)>(StringComparer.Ordinal);
var carried = records.GroupBy(record => record.SchemaId, StringComparer.Ordinal).OrderBy(group => group.Key, StringComparer.Ordinal).ToArray();
foreach (var group in carried)
{
var members = group.ToArray();
var chosen = preferLast ? members[^1] : members[0];
var sameSchema = members.Where(entry => string.Equals(entry.SchemaName, chosen.SchemaName, StringComparison.Ordinal)).ToArray();
if (sameSchema.Length == 1)
var schemaId = group.Key;
var recordKey = group.Min(record => record.Key)!;
var ownRegisteredAll = Canonical(registered.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray();
var ownRegistered = ownRegisteredAll.FirstOrDefault();
var ownArrived = Canonical(arrived.Where(entry => string.Equals(entry.SchemaId, schemaId, StringComparison.Ordinal))).ToArray();
// Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones.
var disagreeing = ownRegisteredAll.Length > 0 ? ownRegisteredAll : ownArrived;
if (disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).Count() > 1)
{
yield return chosen;
continue;
var names = disagreeing.Select(entry => entry.SchemaName).Distinct(StringComparer.Ordinal).ToArray();
throw new CultSchemaConflictException(
$"Schema id '{schemaId}' is claimed by schemas {string.Join(" and ", names.Select(name => $"'{name}'"))}; record '{recordKey}' cannot be written under it.",
schemaId, names, recordKey);
}
yield return new CultSchemaCatalogEntry
CultSchemaCatalogEntry? pick;
bool isRegistered;
if (ownRegistered != null)
{
SchemaId = chosen.SchemaId,
SchemaName = chosen.SchemaName,
SchemaVersion = chosen.SchemaVersion,
ContentHash = chosen.ContentHash,
CanonicalSchemaJson = chosen.CanonicalSchemaJson,
CompatibleSchemaIds = sameSchema.SelectMany(entry => entry.CompatibleSchemaIds).Distinct(StringComparer.Ordinal).ToArray(),
Members = chosen.Members
};
pick = ownRegistered;
isRegistered = true;
}
else if (ownArrived.Length > 0)
{
pick = ownArrived[0];
isRegistered = false;
}
else
{
pick = Canonical(registered.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault();
isRegistered = pick != null;
pick ??= Canonical(arrived.Where(entry => entry.CompatibleSchemaIds.Contains(schemaId, StringComparer.Ordinal))).FirstOrDefault();
}
if (pick == null)
continue;
if (chosen.TryGetValue(pick.SchemaId, out var prior))
{
if (prior.Registered && !isRegistered)
continue;
// Two entries of one tier that share an own id must be one schema: otherwise a record would be read as the other.
if (prior.Registered == isRegistered && !string.Equals(prior.Entry.SchemaName, pick.SchemaName, StringComparison.Ordinal))
{
var names = new[] { prior.Entry.SchemaName, pick.SchemaName };
throw new CultSchemaConflictException(
$"Schema id '{pick.SchemaId}' is claimed by schemas '{names[0]}' and '{names[1]}'; record '{recordKey}' cannot be written under it.",
pick.SchemaId, names, recordKey);
}
}
chosen[pick.SchemaId] = (pick, isRegistered);
}
foreach (var group in carried)
{
if (!chosen.Values.Any(pair => string.Equals(pair.Entry.SchemaId, group.Key, StringComparison.Ordinal) ||
pair.Entry.CompatibleSchemaIds.Contains(group.Key, StringComparer.Ordinal)))
{
var recordKey = group.Min(record => record.Key)!;
throw new CultSchemaConflictException(
$"No schema entry publishes schema id '{group.Key}' of record '{recordKey}'.",
group.Key, chosen.Values.Select(pair => pair.Entry.SchemaName).ToArray(), recordKey);
}
}
return chosen.Values.Select(pair => pair.Entry).ToArray();
}
}
@ -3536,7 +3594,8 @@ namespace GameCult.Caching
ReadSnapshot();
WriteSnapshot(
Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)),
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()),
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()).ToArray(),
Array.Empty<CultSchemaCatalogEntry>(),
Entries.Values.Any(entry => entry.HoldsIds),
existingHeader: null,
wholeStore: true);
@ -3569,9 +3628,6 @@ namespace GameCult.Caching
var records = ontoDisk
? disk.Records.ToDictionary(record => record.Key, StringComparer.Ordinal)
: Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)).ToDictionary(record => record.Key, StringComparer.Ordinal);
var catalog = ontoDisk
? disk.SchemaCatalog
: Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry());
// The committer judged the batch against its own view; the file may have moved since. A merge is judged on the
// set the file will hold, exactly as that set would be judged loading: what other processes wrote (arriving),
// what they removed (departing), and this batch.
@ -3581,6 +3637,20 @@ namespace GameCult.Caching
records.Remove(entry.Key.Value);
foreach (var entry in request.Upserts)
records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload);
if (ontoDisk)
{
// A record this cache holds, unchanged since the file was read, is written under its registered schema's id, as a
// whole-view write writes it: the file's entry that lists an older id for it is then not needed to publish it.
foreach (var record in disk.Records)
{
if (records.TryGetValue(record.Key, out var carried) && ReferenceEquals(carried, record) &&
Entries.TryGetValue(record.Key, out var known) && known.StoredAt == record.StoredAt &&
!string.Equals(known.Descriptor.SchemaId, record.SchemaId, StringComparison.Ordinal))
{
records[record.Key] = ToPersistedRecord(known, SerializePayload);
}
}
}
// Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An
// unconditional commit writes this store's whole view, so what its records hold decides. A record this commit
@ -3588,7 +3658,10 @@ namespace GameCult.Caching
var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal);
var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) ||
(!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value)));
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk);
var registered = Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry())
.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry()))
.ToArray();
WriteSnapshot(records.Values, registered, ontoDisk ? disk.SchemaCatalog : Array.Empty<CultSchemaCatalogEntry>(), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk);
if (!ontoDisk)
Cache?.IdsPersisted(records.Keys);
foreach (var entry in request.Deletes)
@ -3660,17 +3733,15 @@ namespace GameCult.Caching
}
}
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog, bool holdsElementIds, string? existingHeader, bool wholeStore)
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IReadOnlyCollection<CultSchemaCatalogEntry> registered, IReadOnlyCollection<CultSchemaCatalogEntry> arrived, bool holdsElementIds, string? existingHeader, bool wholeStore)
{
var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray();
var used = new HashSet<string>(ordered.Select(record => record.SchemaId), StringComparer.Ordinal);
var snapshot = new CultPersistedStoreSnapshot
{
FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)),
SchemaCatalog = CultSchemaCatalogEntry.MergeById(catalog, preferLast: true)
// An entry stays while it publishes a schema id some record carries, as its own id or a compatible one.
.Where(entry => used.Contains(entry.SchemaId) || entry.CompatibleSchemaIds.Any(used.Contains))
SchemaCatalog = CultSchemaCatalogEntry.Derive(ordered, registered, arrived)
.OrderBy(entry => entry.SchemaName, StringComparer.Ordinal)
.ThenBy(entry => entry.SchemaId, StringComparer.Ordinal)
.ToArray(),
Records = ordered
};

View file

@ -0,0 +1,24 @@
using System;
namespace GameCult.Caching
{
/// <summary>
/// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the
/// records, neither registered), or a record whose schema id no chosen entry publishes. Nothing is written, so the store stays as
/// it was. It names the id, the schema names involved and a record key.
/// </summary>
public sealed class CultSchemaConflictException : InvalidOperationException
{
public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey)
: base(message)
{
SchemaId = schemaId;
SchemaNames = schemaNames;
RecordKey = recordKey;
}
public string SchemaId { get; }
public string[] SchemaNames { get; }
public string RecordKey { get; }
}
}