From 892765ba97824b8de8f16b7a74ec32a2020eeda8 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:24:07 +0000 Subject: [PATCH 1/3] Python: a record with unread slots is held read-only (slot-readonly-py) (#126) --- packages/cultcache-py/CHANGELOG.md | 2 + packages/cultcache-py/README.md | 11 + .../cultcache-py/src/cultcache_py/__init__.py | 2 + .../cultcache-py/src/cultcache_py/cache.py | 113 +++- .../src/cultcache_py/documents.py | 90 +++- .../cultcache-py/src/cultcache_py/stores.py | 16 +- .../cultcache-py/tests/test_slot_readonly.py | 492 ++++++++++++++++++ .../cultcache-py/tests/test_union_parity.py | 8 +- 8 files changed, 704 insertions(+), 30 deletions(-) create mode 100644 packages/cultcache-py/tests/test_slot_readonly.py diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index 95f82042..08fb4d1f 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -9,6 +9,8 @@ ### Breaking +- A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`. + - A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it. - Added `foreign_records()` and the frozen `ForeignRecord(key, schema_id, schema_name, stored_at)`, sorted by key in UTF-16 code-unit order. `stores.retyped` and `stores.written_owns` lose their underscore: the cache judge and the single-file store's commit guard ask one predicate. diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 43e6c90e..72d51562 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -97,6 +97,7 @@ store. - `update_global(...)` - `delete(...)` - `delete_global(...)` +- `is_writable(document, key)` (advisory: would a write or removal of the key be refused; see below) - `snapshot()` ## Indexes And Roles @@ -180,6 +181,16 @@ The payload is a MessagePack array. Field keys are durable slot indexes: - newly added fields should use new keys - fields with defaults tolerate missing or nil slots when older payloads are read +A payload carrying a slot past the slots a reader declares, at any depth (the document, a nested record, a list element, a +union arm, a unit arm), was written by a newer build. It is read, and the record is held read-only: `put`, `update`, +`put_envelope(s)`, `delete` and their global forms refuse its key with `SchemaConflictError` before any store is touched, and +`SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` naming the +first type read with extra slots (never a slot value). `CultCache.is_writable(document, key)` answers the same question before +the write: False for such a record, for a key a store carries foreign, and for a key held as another type's record; the write +still decides. `DocumentDefinition.decode_payload_marked(payload)` returns the value and its `UnreadSlots` (or None). Declaring the +stored id in `compatible_schema_ids` consents to the loss of depth-0 slots that id's catalog entry reports; a nested mark never is. +A store keeps a read-only record's bytes through every other write. A document with a custom `decode` never marks. + That matches the Rust `#[derive(DatabaseEntry)]` formatter shape and the C# `[Key(n)]` intent: schema evolution comes from stable field slots, not source member order. diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index dc73ab1b..4873b6ed 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -9,6 +9,7 @@ from .documents import ( DatabaseEntryField, DocumentDefinition, UnionArm, + UnreadSlots, database_entry_field, define_database_entry_type, define_document_registry, @@ -38,6 +39,7 @@ __all__ = [ "SlotCodec", "StoreUnreadableError", "UnionArm", + "UnreadSlots", "leaves", "database_entry_field", "define_database_entry_type", diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index 315e64ac..85b01d89 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field, replace from typing import Any, Callable, Generic, Iterable, TypeVar from .backing_store import BackingStore, CultCacheEnvelope -from .documents import DocumentDefinition, extract_value +from .documents import DocumentDefinition, UnreadSlots, extract_value from .stores import SchemaConflictError, retyped, written_owns T = TypeVar("T") @@ -123,6 +123,10 @@ class _State: # The records the last pull read that no registered type claims, sorted by key. Derived from the pull and swapped with the # rest of the state, written nowhere else; it decides only refusals at its keys. A store keeps their bytes. foreign: list["ForeignRecord"] = field(default_factory=list) + # The records held read-only, by key (a key holds one record) with the type that holds it: the payload carried slots past the + # declared ones and nothing consents to the loss. Derived from the load or the write that admitted the record and swapped with + # the rest of the state; it decides only refusals at its keys, and any other key is absent. + unread: dict[str, tuple[str, UnreadSlots]] = field(default_factory=dict) @dataclass(frozen=True) @@ -337,6 +341,7 @@ class CultCache: routes, generic = self._state.stores_by_type, self._state.generic_store seen_globals: set[str] = set() foreign: list[ForeignRecord] = [] + loaded_unread: dict[str, tuple[str, UnreadSlots]] = {} type_at_key: dict[str, tuple[str, int]] = {} # one record per key across every store: its type (or schema name) and store stores: list[BackingStore] = [] for candidate in [*self._state.stores_by_type.values(), self._state.generic_store]: @@ -376,8 +381,10 @@ class CultCache: loaded_legacy_global_keys[envelope.type] = envelope.key envelope = replace(envelope, key=GLOBAL_KEY) self._claim_key(type_at_key, envelope.key, envelope.type, position) - envelope = self._stamped(document, envelope) - value = document.decode_payload(envelope.payload) + arrived, envelope = envelope, self._stamped(document, envelope) + value, unread = self._decode_marked(document, arrived) + if unread is not None: + loaded_unread[envelope.key] = (envelope.type, unread) loaded_values.setdefault(envelope.type, {})[envelope.key] = value read_values[(envelope.type, envelope.key)] = value loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope @@ -393,6 +400,7 @@ class CultCache: self._state.lookup_keys, self._state.legacy_global_keys, self._state.foreign, + self._state.unread, ) = ( loaded_values, loaded_envelopes, @@ -401,6 +409,7 @@ class CultCache: loaded_lookup_keys, loaded_legacy_global_keys, sorted(foreign, key=lambda record: _utf16_order(record.key)), + loaded_unread, ) @staticmethod @@ -491,9 +500,9 @@ class CultCache: def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T: self._assert_registered(document) self._check_envelope(document, envelope) - envelope = self._stamped(document, envelope) - value = document.decode_payload(envelope.payload) - self._write(document, [(envelope, value)], batch=False) + arrived, envelope = envelope, self._stamped(document, envelope) + value, unread = self._decode_marked(document, envelope) + self._write(document, [(envelope, value)], batch=False, unread={envelope.key: unread}) return value @_mutating @@ -501,13 +510,33 @@ class CultCache: self._assert_registered(document) values: list[T] = [] stamped: list[CultCacheEnvelope] = [] + unread: dict[str, UnreadSlots | None] = {} for envelope in envelopes: self._check_envelope(document, envelope) stamped.append(self._stamped(document, envelope)) - values.append(document.decode_payload(envelope.payload)) - self._write(document, list(zip(stamped, values)), batch=True) + value, unread[envelope.key] = self._decode_marked(document, stamped[-1]) + values.append(value) + self._write(document, list(zip(stamped, values)), batch=True, unread=unread) return values + @staticmethod + def _decode_marked(document: DocumentDefinition[Any], held_as: CultCacheEnvelope) -> tuple[Any, UnreadSlots | None]: + """The payload decoded under the document, with the mark of any slot it carried past the declared ones, or no mark when the + loss is consented. `held_as` is the envelope under the id the record is stored under: a load's arrival, a write's stamped + envelope, so a key is writable in memory exactly when it is after a reload. A depth-0 mark is consented only when that id is + one the document lists compatible and its catalog entry has slots past the document's that cover the mark's slot count (the + loss a compatible-id declaration consents to, cultlib-gaps:ruling:slot-mark-consent). Any other mark keeps the record read-only.""" + value, slots = document.decode_payload_marked(held_as.payload) + if slots is None: + return value, None + entry, stored = document.catalog_entry(), held_as.catalog_entry + if slots.depth0_only and stored is not None and held_as.schema_id in entry.compatible_schema_ids and held_as.schema_id != entry.schema_id: + declared = {member.slot for member in entry.members} + reported = [member.slot for member in stored.members if member.slot not in declared] + if reported and slots.slot_count <= max(reported) + 1: + return value, None + return value, slots + @staticmethod def _stamped(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> CultCacheEnvelope: """The record under its registered schema's id, whatever id the envelope arrived under, so the catalog entry a store @@ -527,7 +556,12 @@ class CultCache: raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}") def _write( - self, document: DocumentDefinition[Any], records: list[tuple[CultCacheEnvelope, Any]], *, batch: bool + self, + document: DocumentDefinition[Any], + records: list[tuple[CultCacheEnvelope, Any]], + *, + batch: bool, + unread: dict[str, UnreadSlots | None] | None = None, ) -> None: """Validates every record (home and name and index extractors included) before the store is touched; once the store accepts, applying to the cache has no fallible step.""" @@ -535,6 +569,7 @@ class CultCache: # A batch naming a key twice lands the last record for it, as C# CultCacheBatch stages one operation per key. The # judge, the store and the apply all read this one list. records = list({envelope.key: (envelope, value) for envelope, value in records}.values()) + self._refuse_unread([envelope.key for envelope, _ in records]) name_extractor = self._state.name_extractors.get(type) index_extractors = self._state.index_extractors.get(type, {}) admitted = [ @@ -566,6 +601,9 @@ class CultCache: self._remove_lookups(type, envelope.key) values[envelope.key] = value envelopes[envelope.key] = envelope + # A held mark refused this write above, so a key reaching here carries none to clear. + if (mark := (unread or {}).get(envelope.key)) is not None: + self._state.unread[envelope.key] = (type, mark) lookup_keys[envelope.key] = keys if keys.name is not None: self._state.names[(type, keys.name)] = envelope.key @@ -592,6 +630,51 @@ class CultCache: replaced.append((other, envelope.key)) return replaced + @_locked + def is_writable(self, document: DocumentDefinition[Any], key: str) -> bool: + """Whether a write or removal of the key by this document would pass the judges that run before any store is touched: False for + a record held read-only (its payload carried slots past the declared ones, at any depth, and nothing consents to the loss, so + writing it would drop data a newer writer stored), for a key a store carries foreign, and for a key held as another type's + record this document does not own. Advisory (the write still decides): it answers at call time, and a key nothing holds is + writable.""" + self._assert_registered(document) + try: + self._refuse_at(document, key) + except (SchemaConflictError, CultCacheError): + return False + return True + + def _refuse_at(self, document: DocumentDefinition[Any], key: str) -> list[tuple[str, str]]: + """The judges a write or removal of the key by this document passes before any store is touched, asked by `is_writable` and + by `delete`: a global document writes only its global key, a record held read-only is never replaced, and a key held as + another type's record or carried foreign is refused (`_refuse_retype`).""" + if document.global_document and key != GLOBAL_KEY: + raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}") + self._refuse_unread([key]) + entry = document.catalog_entry() + probe = CultCacheEnvelope.create(key=key, type=document.type, payload=b"", schema_id=entry.schema_id, catalog_entry=entry) + return self._refuse_retype(document, [probe]) + + def _refuse_unread(self, keys: list[str]) -> None: + """The one check every write and removal asks before any store is touched.""" + for key in keys: + held = self._state.unread.get(key) + if held is None: + continue + type, slots = held + raise SchemaConflictError( + self._state.envelopes[type][key].schema_id or type, + [self._state.documents[type].catalog_entry().schema_name], + key, + message=( + f"{self._state.documents[type].catalog_entry().schema_name} record {key} is read-only: {slots.type_name} was read with " + f"{slots.slot_count} slots where this build declares {slots.declared_count} ({slots.occurrences} object(s) in the " + "record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or " + "write it from that build." + ), + unread_slots=slots, + ) + @staticmethod def _foreign_refusal(record: ForeignRecord) -> SchemaConflictError: return SchemaConflictError( @@ -644,16 +727,18 @@ class CultCache: def _delete(self, document: DocumentDefinition[Any], key: str) -> None: self._assert_registered(document) - if key not in self._state.envelopes.get(document.type, {}): - carried = next((record for record in self._state.foreign if record.key == key), None) - if carried is not None: - raise self._foreign_refusal(carried) + # A key held as another type's record this document owns is removed, as a write of the key would replace it (C# Remove is by key). + owned = self._refuse_at(document, key) store = self._store_for_type(document.type) legacy_key = self._state.legacy_global_keys.get(document.type) if key == GLOBAL_KEY else None if store is not None: - store.delete(document.type, key if legacy_key is None else legacy_key) + # One record per key: the owned record when this document does not hold the key, else its own, in one store call. + holder = owned[0][0] if owned else document.type + store.delete(holder, key if legacy_key is None else legacy_key) if legacy_key is not None: del self._state.legacy_global_keys[document.type] + for other, _ in owned: + self._drop(other, key) self._drop(document.type, key) def _drop(self, type: str, key: str) -> None: diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 6101c136..4f7dfe7d 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import threading from dataclasses import MISSING, dataclass, field from collections.abc import Mapping, Sequence, Set from itertools import chain @@ -30,6 +31,49 @@ MAX_SLOT_KEY = 65535 _STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,) +@dataclass(frozen=True) +class UnreadSlots: + """What a payload decode found past the slots this runtime declares, at any depth: the first type read with extra slots, how + many slots it carried against how many it declares, and how many objects in the payload did. It holds no slot value, so a + refusal can name it. `depth0_only` is True when every mark sat in the document's own record (the half of consent the payload + decides). A record carrying it is read-only unless the schema comparison explains every mark (see `CultCache.is_writable`).""" + + type_name: str + slot_count: int + declared_count: int + occurrences: int + depth0_only: bool + + +class _SlotScope: + """Opened by `DocumentDefinition.decode_payload_marked`; the record codec and the union codec note into it. It is thread-local, + so no codec signature carries it, and a codec decoded outside a payload decode notes nothing.""" + + def __init__(self) -> None: + self.depth = 0 # records entered: 0 while the document's own record is read + self.first: tuple[str, int, int] | None = None + self.occurrences = 0 + self.depth0_only = True + + def note(self, type_name: str, slot_count: int, declared_count: int, depth0: bool) -> None: + if self.first is None: + self.first = (type_name, slot_count, declared_count) + self.occurrences += 1 + self.depth0_only = self.depth0_only and depth0 + + def result(self) -> UnreadSlots | None: + if self.first is None: + return None + return UnreadSlots(*self.first, self.occurrences, self.depth0_only) + + +_scope = threading.local() + + +def _current_scope() -> _SlotScope | None: + return getattr(_scope, "open", None) + + _CONTAINERS = (Mapping, Sequence, Set) _SCALAR_SEQUENCES = (str, bytes, bytearray, memoryview) @@ -185,15 +229,25 @@ class DocumentDefinition(Generic[T]): return self.payload_encoder(tree) def decode_payload(self, payload: bytes) -> T: + return self.decode_payload_marked(payload)[0] + + def decode_payload_marked(self, payload: bytes) -> tuple[T, UnreadSlots | None]: + """The value and what the decode found past the declared slots, at any depth. The payload depth check runs first: a + payload past the limit is refused, never marked.""" try: tree = self.payload_decoder(payload) except _STACK_FAULTS: raise CultCachePayloadDepthError(self.type, "read") from None _refuse_past_depth_limit(tree, self.type, "read") - value = self.decode(tree) + outer, opened = _current_scope(), _SlotScope() + _scope.open = opened + try: + value = self.decode(tree) + finally: + _scope.open = outer if self.validate: self.validate(value) - return value + return value, opened.result() def __post_init__(self) -> None: # Judged here so every constructor passes it: the factories, the dataclass constructor and dataclasses.replace. @@ -418,6 +472,9 @@ def define_union( raise ValueError(f"Union {name!r} value is malformed: expected [armKey, [armSlots]]") arm_name, codec = by_key[key] if codec is None: + scope = _current_scope() + if scope is not None and raw[1]: + scope.note(arm_name, len(raw[1]), 0, False) return UnionArm(arm_name, None) try: return UnionArm(arm_name, codec.decode(raw[1])) @@ -479,17 +536,26 @@ def _record_codec( def decode(raw: Any) -> Any: if not isinstance(raw, list): raise ValueError(f"DatabaseEntry document type {type!r} expected a MessagePack array") + scope = _current_scope() + if scope is not None and len(raw) > max_key + 1: + scope.note(type, len(raw), max_key + 1, scope.depth == 0) values: dict[str, Any] = {} - for field in normalized: - if field.key < len(raw): - try: - values[field.name] = field.codec.decode(raw[field.key]) - except ValueError as error: - raise ValueError(f"{type} slot {field.key} ({field.name}): {error}") from None - elif field.default is MISSING: - values[field.name] = None - else: - values[field.name] = field.default() if callable(field.default) else field.default + if scope is not None: + scope.depth += 1 + try: + for field in normalized: + if field.key < len(raw): + try: + values[field.name] = field.codec.decode(raw[field.key]) + except ValueError as error: + raise ValueError(f"{type} slot {field.key} ({field.name}): {error}") from None + elif field.default is MISSING: + values[field.name] = None + else: + values[field.name] = field.default() if callable(field.default) else field.default + finally: + if scope is not None: + scope.depth -= 1 if cls is None: return values return cls(**values) diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 61841956..c15ac32e 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Iterator, TypeVar import msgpack +from .documents import UnreadSlots from .backing_store import ( CultCacheEnvelope, CultCacheSchemaCatalogEntry, @@ -46,9 +47,19 @@ class SchemaConflictError(ValueError): """A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names involved (`schema_names`) and a record key (`record_key`). A cache raises it too, with an empty record key, when a - document registers a schema id another registered document already carries.""" + document registers a schema id another registered document already carries. `unread_slots` is None except for the refusal to + write or remove a record held read-only because its payload carries slots past the declared ones; its message names the type + and both slot counts and never holds a slot value.""" - def __init__(self, schema_id: str, schema_names: list[str], record_key: str, *, message: str | None = None) -> None: + def __init__( + self, + schema_id: str, + schema_names: list[str], + record_key: str, + *, + message: str | None = None, + unread_slots: UnreadSlots | None = None, + ) -> None: super().__init__( message or f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}" @@ -56,6 +67,7 @@ class SchemaConflictError(ValueError): self.schema_id = schema_id self.schema_names = schema_names self.record_key = record_key + self.unread_slots = unread_slots class JsonLinesBackingStore: diff --git a/packages/cultcache-py/tests/test_slot_readonly.py b/packages/cultcache-py/tests/test_slot_readonly.py new file mode 100644 index 00000000..77d4a989 --- /dev/null +++ b/packages/cultcache-py/tests/test_slot_readonly.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import dataclasses +import shutil +import threading +import tempfile +import unittest +from pathlib import Path + +import msgpack +from cultcache_py.cache import CultCacheError + +from cultcache_py import ( + CultCache, + CultCachePayloadDepthError, + SlotCodec, + SchemaConflictError, + SingleFileMessagePackBackingStore, + UnionArm, + UnreadSlots, + database_entry_field, + define_database_entry_type, + define_record_codec, + leaves as t, +) +import test_union_parity as _parity +from test_union_parity import ( + VECTORS, + _surface_value, + _vector, + surface_document, + unit_outcome, +) + +SLOT = "extra-slot" # the canary: the value of every extra slot in the vectors; no refusal may carry it + +# The vectors name the C# types; this runtime names a type by its codec's name, a document by its document type. +_NAMES = {"UnionSurface": surface_document.type, "SurfaceDeclaring": surface_document.type, "Denied2": "Denied"} + +slot_unit_document = define_database_entry_type( + "tests.slot_unit", + [database_entry_field("name", 0, codec=t.string), database_entry_field("outcome", 1, codec=unit_outcome)], + name="name", +) +# The reader of extra-slot-consent.cc: the surface document's slots under a new schema id that lists the old one compatible. +declaring_document = define_database_entry_type( + "tests.union_surface", + [ + database_entry_field("id", 0, codec=t.string), + database_entry_field("version", 1, codec=t.int32), + database_entry_field("root", 2, codec=_parity._node), + database_entry_field("outcomes", 3, codec=t.list(_parity.outcome)), + database_entry_field("part", 4, codec=t.nullable(_parity.item)), + database_entry_field("crafted", 5, codec=t.nullable(_parity.crafted)), + database_entry_field("missing", 6, codec=_parity._node), + ], + name="id", + schema_id="tests.union_surface.v2", + schema_name="tests.union_surface", + compatible_schema_ids=("tests.union_surface.wide",), +) + +# file -> (type, slots carried, slots declared, every mark at depth 0): the README table. +_PAYLOADS = { + "extra-slot-record.msgpack": ("UnionSurface", 8, 7, True), + "extra-slot-nested-record.msgpack": ("Composition", 3, 2, False), + "extra-slot-arm.msgpack": ("Box", 3, 2, False), + "extra-slot-list-element.msgpack": ("Text", 3, 2, False), + "depth-skipped-at-limit.msgpack": ("UnionSurface", 8, 7, True), +} + + +def _copy_store_without_variants(source: Path, target: Path) -> None: + """This runtime holds no variant records (a five-slot record makes a store unreadable here), so the store vector is + walked with its variant row removed; every other byte is the C# writer's.""" + [snapshot] = list(msgpack.Unpacker(open(source, "rb"), raw=False)) + snapshot[2] = [record for record in snapshot[2] if len(record) <= 4] + target.write_bytes(msgpack.packb(snapshot, use_bin_type=True)) + + +def _open(path: Path, *documents) -> CultCache: + builder = CultCache.builder() + for document in documents: + builder.register_document_type(document) + cache = builder.add_generic_store(SingleFileMessagePackBackingStore(path)).build() + cache.pull_all_backing_stores() + return cache + + +def _rows(): + return msgpack.unpackb(_vector("slot-outcomes.msgpack"), raw=False) + + +class PayloadMarkTests(unittest.TestCase): + def test_every_read_only_payload_decodes_and_names_its_mark(self) -> None: + for file, (type_name, carried, declared, depth0) in _PAYLOADS.items(): + with self.subTest(file=file): + value, mark = surface_document.decode_payload_marked(_vector(file)) + self.assertEqual(value, _surface_value()) + self.assertEqual(mark, UnreadSlots(_NAMES.get(type_name, type_name), carried, declared, 1, depth0)) + self.assertEqual(surface_document.decode_payload(_vector(file)), _surface_value()) + + def test_the_outcomes_manifest_still_counts_nine_rows_and_four_read_only(self) -> None: + rows = msgpack.unpackb(_vector("outcomes.msgpack"), raw=False) + self.assertEqual(len(rows), 9) + self.assertEqual([row[0] for row in rows if row[1] == "read-only"], [f for f in _PAYLOADS if f != "depth-skipped-at-limit.msgpack"]) + + def test_a_clean_payload_and_a_short_one_carry_no_mark(self) -> None: + self.assertEqual(surface_document.decode_payload_marked(_vector("union-surface.msgpack")), (_surface_value(), None)) + slots = msgpack.unpackb(_vector("union-surface.msgpack"), raw=False) + slots[2] = [1, ["only-id"]] # a Text arm with a missing trailing slot: defaults, no mark + _, mark = surface_document.decode_payload_marked(msgpack.packb(slots, use_bin_type=True)) + self.assertIsNone(mark) + + def test_a_retired_slot_below_the_highest_key_does_not_mark(self) -> None: + slots = msgpack.unpackb(_vector("union-surface.msgpack"), raw=False) + # Image's declared slots are 0, 1, 3 (2 retired): four carried, four declared. + image = slots[2][1][1][2] + self.assertEqual(image[0], 5) + image[1][2] = "retired-slot-value" + _, mark = surface_document.decode_payload_marked(msgpack.packb(slots, use_bin_type=True)) + self.assertIsNone(mark) + + def test_the_depth_check_runs_before_the_mark(self) -> None: + with self.assertRaises(CultCachePayloadDepthError): + surface_document.decode_payload_marked(_vector("depth-skipped-past-limit.msgpack")) + + def test_the_unit_arm_with_extra_slots_is_read_past_and_marked(self) -> None: + payload = msgpack.packb(["unit-arm", [2, [SLOT]]], use_bin_type=True) + value, mark = slot_unit_document.decode_payload_marked(payload) + self.assertEqual(value, {"name": "unit-arm", "outcome": UnionArm("Denied")}) + self.assertEqual(mark, UnreadSlots("Denied", 1, 0, 1, False)) + self.assertEqual(slot_unit_document.decode_payload_marked(msgpack.packb(["ok", [2, []]]))[1], None) + + def test_a_codec_decoded_outside_a_payload_decode_notes_nothing(self) -> None: + self.assertEqual(unit_outcome.decode([2, [SLOT]]), UnionArm("Denied")) + + def test_the_scope_belongs_to_its_thread(self) -> None: + def decode_elsewhere(raw): + thread = threading.Thread(target=lambda: unit_outcome.decode([2, [SLOT]])) + thread.start() + thread.join() + return raw + + outer = define_database_entry_type("tests.threaded", [database_entry_field("a", 0, codec=SlotCodec(lambda v: v, decode_elsewhere))]) + self.assertIsNone(outer.decode_payload_marked(msgpack.packb(["x"]))[1]) + + def test_a_nested_payload_decode_restores_the_outer_scope(self) -> None: + def decode_clean_inside(raw): + surface_document.decode_payload_marked(_vector("union-surface.msgpack")) + return raw + + inner = define_record_codec("Inner", [database_entry_field("x", 0, codec=t.string)]) + outer = define_database_entry_type( + "tests.nested", + [ + database_entry_field("a", 0, codec=SlotCodec(lambda v: v, decode_clean_inside)), + database_entry_field("b", 1, codec=inner), + ], + ) + _, mark = outer.decode_payload_marked(msgpack.packb(["x", ["v", SLOT]])) + self.assertEqual(mark, UnreadSlots("Inner", 2, 1, 1, False)) + + +class StoreRowTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + + def _store(self, name: str) -> tuple[Path, CultCache]: + path = self.tmp / name + _copy_store_without_variants(VECTORS / name, path) + documents = (declaring_document,) if name == "extra-slot-consent.cc" else (surface_document, slot_unit_document) + return path, _open(path, *documents) + + def _document(self, store: str, key: str): + if key == "unit-arm": + return slot_unit_document + return declaring_document if store == "extra-slot-consent.cc" else surface_document + + def test_every_row_is_writable_exactly_where_the_vector_says_and_refuses_with_its_mark(self) -> None: + walked = 0 + for store, key, writable, explained, type_name, carried, declared, occurrences, flatten in _rows(): + if flatten is not None: + continue # a variant: this runtime holds none (deviation) + walked += 1 + with self.subTest(store=store, key=key): + path, cache = self._store(store) + document = self._document(store, key) + before = path.read_bytes() + self.assertEqual(cache.is_writable(document, key), writable) + held = cache.get(document, key) + self.assertIsNotNone(held) + if writable: + continue + self.assertEqual(explained, False) + for attempt in ( + lambda: cache.put(document, key, held), + lambda: cache.delete(document, key), + lambda: cache.put_envelope(document, cache.get_required_envelope(document, key)), + lambda: cache.put_envelopes(document, [cache.get_required_envelope(document, key)]), + lambda: cache.update(document, key, lambda value: value), + ): + with self.assertRaises(SchemaConflictError) as caught: + attempt() + refusal = caught.exception + self.assertEqual( + refusal.unread_slots, + UnreadSlots(_NAMES.get(type_name, type_name), carried, declared, occurrences, refusal.unread_slots.depth0_only), + ) + self.assertEqual(refusal.record_key, key) + message = str(refusal) + self.assertNotIn(SLOT, message) + self.assertIn(refusal.unread_slots.type_name, message) + self.assertIn(f"{carried} slots", message) + self.assertIn(f"declares {declared}", message) + self.assertEqual(path.read_bytes(), before) + self.assertTrue(cache.get(document, key) is held) + self.assertEqual(walked, 7) # eight rows, the variant not walked + + def test_a_key_no_store_holds_and_another_key_are_writable_while_a_marked_one_is_held(self) -> None: + path, cache = self._store("extra-slot-store.cc") + self.assertFalse(cache.is_writable(surface_document, "marked")) + self.assertTrue(cache.is_writable(surface_document, "nothing-here")) + cache.put(surface_document, "other", cache.get_required(surface_document, "clean")) + self.assertEqual(cache.get(surface_document, "other"), _surface_value()) + + def test_consent_lands_an_explained_mark_and_refuses_the_others(self) -> None: + path, cache = self._store("extra-slot-consent.cc") + self.assertTrue(cache.is_writable(declaring_document, "consented")) + cache.put(declaring_document, "consented", cache.get_required(declaring_document, "consented")) + [stored] = [e for e in SingleFileMessagePackBackingStore(path).pull_all() if e.key == "consented"] + self.assertEqual(stored.schema_id, "tests.union_surface.v2") + self.assertEqual(len(msgpack.unpackb(stored.payload, raw=False)), 7) # the consented slot is gone + self.assertTrue(cache.is_writable(declaring_document, "consented")) + for key in ("over-consent", "nested"): + self.assertFalse(cache.is_writable(declaring_document, key), key) + with self.assertRaises(SchemaConflictError): + cache.put(declaring_document, key, cache.get_required(declaring_document, key)) + + def test_a_reload_marks_again_and_a_write_of_a_clean_envelope_over_a_marked_key_is_refused(self) -> None: + path, cache = self._store("extra-slot-store.cc") + envelope = cache.get_required_envelope(surface_document, "clean") + with self.assertRaises(SchemaConflictError): + cache.put_envelope(surface_document, type(envelope)(**{**envelope.__dict__, "key": "marked"})) + cache.pull_all_backing_stores() + self.assertFalse(cache.is_writable(surface_document, "marked")) + + def test_a_written_envelope_that_carries_extra_slots_is_held_read_only(self) -> None: + path, cache = self._store("extra-slot-store.cc") + envelope = cache.get_required_envelope(surface_document, "clean") + carrying = type(envelope)(**{**envelope.__dict__, "key": "arrived", "payload": _vector("extra-slot-arm.msgpack")}) + cache.put_envelope(surface_document, carrying) + self.assertFalse(cache.is_writable(surface_document, "arrived")) + with self.assertRaises(SchemaConflictError): + cache.delete(surface_document, "arrived") + cache.pull_all_backing_stores() + self.assertFalse(cache.is_writable(surface_document, "arrived")) + + def test_a_batch_of_envelopes_that_carry_extra_slots_is_held_read_only(self) -> None: + path, cache = self._store("extra-slot-store.cc") + envelope = cache.get_required_envelope(surface_document, "clean") + batch = [ + type(envelope)(**{**envelope.__dict__, "key": key, "payload": _vector(file)}) + for key, file in (("b1", "extra-slot-arm.msgpack"), ("b2", "union-surface.msgpack")) + ] + cache.put_envelopes(surface_document, batch) + self.assertEqual([cache.is_writable(surface_document, key) for key in ("b1", "b2")], [False, True]) + + def test_a_mark_under_an_id_the_reader_does_not_list_compatible_is_not_consented(self) -> None: + unlisted = define_database_entry_type( + "tests.union_surface", + [database_entry_field("id", 0, codec=t.string)] + [database_entry_field(n, i, codec=c) for n, i, c in ( + ("version", 1, t.int32), ("root", 2, _parity._node), ("outcomes", 3, t.list(_parity.outcome)), + ("part", 4, t.nullable(_parity.item)), ("crafted", 5, t.nullable(_parity.crafted)), ("missing", 6, _parity._node))], + name="id", + schema_id="tests.union_surface.v2", + schema_name="tests.union_surface", + ) + path = self.tmp / "unlisted.cc" + _copy_store_without_variants(VECTORS / "extra-slot-consent.cc", path) + cache = _open(path, unlisted) + self.assertFalse(cache.is_writable(unlisted, "consented")) + + def test_a_written_envelope_is_judged_on_the_id_it_is_stored_under_so_a_reload_agrees(self) -> None: + path, cache = self._store("extra-slot-consent.cc") + arrived = next(e for e in SingleFileMessagePackBackingStore(path).pull_all() if e.key == "consented") + self.assertEqual(arrived.schema_id, "tests.union_surface.wide") + cache.put_envelope(declaring_document, dataclasses.replace(arrived, key="via-envelope")) + cache.put_envelopes(declaring_document, [dataclasses.replace(arrived, key="via-batch")]) + in_memory = [cache.is_writable(declaring_document, k) for k in ("via-envelope", "via-batch")] + cache.pull_all_backing_stores() + self.assertEqual([cache.is_writable(declaring_document, k) for k in ("via-envelope", "via-batch")], in_memory) + self.assertEqual(in_memory, [False, False]) + + def test_is_writable_agrees_with_the_write_for_a_key_a_store_carries_foreign(self) -> None: + path = self.tmp / "foreign.cc" + _copy_store_without_variants(VECTORS / "extra-slot-store.cc", path) + cache = _open(path, surface_document) # tests.slot_unit is not registered: its record is carried foreign + self.assertEqual([r.key for r in cache.foreign_records()], ["unit-arm"]) + self.assertFalse(cache.is_writable(surface_document, "unit-arm")) + with self.assertRaises(SchemaConflictError): + cache.put(surface_document, "unit-arm", _surface_value()) + with self.assertRaises(SchemaConflictError): + cache.delete(surface_document, "unit-arm") + + def test_every_key_of_a_batch_is_checked_wherever_the_marked_one_stands(self) -> None: + path, cache = self._store("extra-slot-store.cc") + before = path.read_bytes() + envelope = cache.get_required_envelope(surface_document, "clean") + fresh = dataclasses.replace(envelope, key="fresh") + marked = dataclasses.replace(envelope, key="marked") + for batch in ([fresh, marked], [marked, fresh], [fresh, dataclasses.replace(envelope, key="f2"), marked]): + with self.assertRaises(SchemaConflictError) as caught: + cache.put_envelopes(surface_document, batch) + self.assertEqual(caught.exception.record_key, "marked") + self.assertEqual(path.read_bytes(), before) + self.assertIsNone(cache.get(surface_document, "fresh")) + + def test_only_the_marks_that_hold_a_record_read_only_are_kept_by_key(self) -> None: + _, store = self._store("extra-slot-store.cc") + self.assertEqual({k: v[0] for k, v in store._state.unread.items()}, {"marked": surface_document.type, "unit-arm": "tests.slot_unit"}) + _, consent = self._store("extra-slot-consent.cc") + self.assertEqual(sorted(consent._state.unread), ["nested", "over-consent"]) # the consented mark is no state + + def test_an_arrival_with_no_catalog_entry_under_a_listed_id_is_not_consented(self) -> None: + wide = next(e for e in SingleFileMessagePackBackingStore(VECTORS / "extra-slot-consent.cc").pull_all() if e.key == "consented") + bare = dataclasses.replace(wide, catalog_entry=None) + + class Held: + def pull_all(self): + return [bare] + + def push(self, envelope): ... + + def push_all(self, envelopes): ... + + def delete(self, type, key): ... + + cache = CultCache.builder().register_document_type(declaring_document).add_generic_store(Held()).build() + cache.pull_all_backing_stores() + self.assertFalse(cache.is_writable(declaring_document, "consented")) + + def test_a_removal_is_refused_where_is_writable_says_no(self) -> None: + path, cache = self._store("extra-slot-store.cc") + before = path.read_bytes() + other = define_database_entry_type("tests.other", [database_entry_field("a", 0, codec=t.string)], name="a") + cache2 = _open(self.tmp / "extra-slot-store.cc", surface_document, slot_unit_document, other) + self.assertFalse(cache2.is_writable(other, "clean")) + with self.assertRaises(SchemaConflictError): + cache2.delete(other, "clean") + self.assertEqual(path.read_bytes(), before) + self.assertIsNotNone(cache2.get(surface_document, "clean")) + self.assertTrue(cache2.is_writable(other, "nothing-here")) + + def test_a_global_document_is_not_writable_at_a_key_other_than_the_global_one(self) -> None: + solo = define_database_entry_type("tests.solo", [database_entry_field("a", 0, codec=t.string)], global_document=True) + cache = CultCache.builder().register_document_type(solo).build() + self.assertFalse(cache.is_writable(solo, "elsewhere")) + with self.assertRaises(CultCacheError): + cache.put(solo, "elsewhere", {"a": "x"}) + with self.assertRaises(CultCacheError): + cache.delete(solo, "elsewhere") + self.assertTrue(cache.is_writable(solo, CultCache.GLOBAL_KEY)) + + def test_a_reload_replaces_the_marks_wholesale(self) -> None: + path, cache = self._store("extra-slot-store.cc") + self.assertFalse(cache.is_writable(surface_document, "marked")) + # The store is repaired by its owner: the marked record is rewritten clean. The reload must forget the old mark. + [snapshot] = list(msgpack.Unpacker(open(path, "rb"), raw=False)) + clean = next(r for r in snapshot[2] if r[0] == "clean") + for record in snapshot[2]: + if record[0] in ("marked", "unit-arm"): + record[3] = clean[3] if record[0] == "marked" else record[3] + snapshot[2] = [r for r in snapshot[2] if r[0] != "unit-arm"] + path.write_bytes(msgpack.packb(snapshot, use_bin_type=True)) + cache.pull_all_backing_stores() + self.assertTrue(cache.is_writable(surface_document, "marked")) + self.assertEqual(cache._state.unread, {}) + + def test_a_listed_id_whose_catalog_reports_no_extra_slot_does_not_consent(self) -> None: + wide = next(e for e in SingleFileMessagePackBackingStore(VECTORS / "extra-slot-consent.cc").pull_all() if e.key == "consented") + same = dataclasses.replace(wide, catalog_entry=declaring_document.catalog_entry()) # the stored entry has no slot past the reader's + + class Held: + def pull_all(self): + return [same] + + def push(self, envelope): ... + + def push_all(self, envelopes): ... + + def delete(self, type, key): ... + + cache = CultCache.builder().register_document_type(declaring_document).add_generic_store(Held()).build() + cache.pull_all_backing_stores() + self.assertFalse(cache.is_writable(declaring_document, "consented")) + + def test_a_record_stored_under_the_readers_own_id_is_never_consented(self) -> None: + own = dataclasses.replace( + next(e for e in SingleFileMessagePackBackingStore(VECTORS / "extra-slot-consent.cc").pull_all() if e.key == "consented"), + schema_id="tests.union_surface.v2", + ) + wider = dataclasses.replace(own.catalog_entry, schema_id="tests.union_surface.v2", compatible_schema_ids=("tests.union_surface.v2",)) + listing = dataclasses.replace( + declaring_document, schema_id="tests.union_surface.v2", compatible_schema_ids=("tests.union_surface.wide", "tests.union_surface.v2") + ) + + class Held: + def pull_all(self): + return [dataclasses.replace(own, catalog_entry=wider)] + + def push(self, envelope): ... + + def push_all(self, envelopes): ... + + def delete(self, type, key): ... + + cache = CultCache.builder().register_document_type(listing).add_generic_store(Held()).build() + cache.pull_all_backing_stores() + self.assertFalse(cache.is_writable(listing, "consented")) + + def test_a_removal_by_the_owner_of_a_key_held_as_another_type_removes_it_as_a_write_would_replace_it(self) -> None: + a = define_database_entry_type( + "tests.owned_a", [database_entry_field("a", 0, codec=t.string)], name="a", indexes={"by_a": "a"} + ) + b = define_database_entry_type( + "tests.owned_b", + [database_entry_field("a", 0, codec=t.string)], + compatible_schema_ids=(a.catalog_entry().schema_id,), + ) + path = self.tmp / "owned.cc" + writer = _open(path, a) + writer.put(a, "k", {"a": "x"}) + writer.put(a, "other", {"a": "y"}) + cache = _open(path, a, b) + self.assertTrue(cache.is_writable(b, "k")) + cache.delete(b, "k") + self.assertIsNone(cache.get(a, "k")) + self.assertIsNone(cache.get_by_index(a, "by_a", "x")) + self.assertEqual([e.key for e in SingleFileMessagePackBackingStore(path).pull_all()], ["other"]) + self.assertTrue(cache.is_writable(b, "k")) + + def test_an_owners_removal_is_one_store_call_so_the_cache_and_the_store_agree_when_it_fails(self) -> None: + a = define_database_entry_type("tests.one_a", [database_entry_field("a", 0, codec=t.string)], name="a") + b = define_database_entry_type( + "tests.one_b", [database_entry_field("a", 0, codec=t.string)], compatible_schema_ids=(a.catalog_entry().schema_id,) + ) + path = self.tmp / "one.cc" + _open(path, a).put(a, "k", {"a": "x"}) + + class SecondDeleteFails(SingleFileMessagePackBackingStore): + calls = 0 + + def delete(self, type, key): + self.calls += 1 + if self.calls > 1: + raise OSError("second delete") + super().delete(type, key) + + store = SecondDeleteFails(path) + cache = CultCache.builder().register_document_type(a).register_document_type(b).add_generic_store(store).build() + cache.pull_all_backing_stores() + cache.delete(b, "k") + self.assertEqual(store.calls, 1) + self.assertIsNone(cache.get(a, "k")) + self.assertEqual(SingleFileMessagePackBackingStore(path).pull_all(), []) + + def test_writing_another_record_and_flushing_carries_the_marked_one_unchanged(self) -> None: + path, cache = self._store("extra-slot-store.cc") + original = {e.key: e for e in SingleFileMessagePackBackingStore(path).pull_all()} + cache.put(surface_document, "clean", {**_surface_value(), "version": 9}) + after = {e.key: e for e in SingleFileMessagePackBackingStore(path).pull_all()} + for key in ("marked", "unit-arm"): + self.assertEqual(after[key].payload, original[key].payload, key) + self.assertEqual(after[key].schema_id, original[key].schema_id, key) + self.assertEqual(after[key].stored_at, original[key].stored_at, key) + self.assertNotEqual(after["clean"].payload, original["clean"].payload) + self.assertEqual(after["marked"].catalog_entry, original["marked"].catalog_entry) + + def test_the_vector_store_loads_under_the_reference_documents_and_marks_nothing_else(self) -> None: + path, cache = self._store("extra-slot-store.cc") + self.assertTrue(cache.is_writable(surface_document, "clean")) + self.assertFalse(cache.is_writable(surface_document, "marked")) + self.assertFalse(cache.is_writable(slot_unit_document, "unit-arm")) + self.assertEqual( + cache.get(slot_unit_document, "unit-arm"), {"name": "unit-arm", "outcome": UnionArm("Denied")} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index 451cd030..1e19ce8d 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -261,7 +261,9 @@ class UnionSurfaceVectorTests(unittest.TestCase): else: read_only += 1 self.assertEqual(outcome_kind, "read-only", file) - self.assertEqual(surface_document.decode_payload(payload), _surface_value(), file) + value, mark = surface_document.decode_payload_marked(payload) + self.assertEqual(value, _surface_value(), file) + self.assertIsNotNone(mark, file) # read past and marked: tests/test_slot_readonly.py pins what the mark holds self.assertEqual((refused, read_only), (5, 4)) @@ -526,7 +528,9 @@ class BoundsVectorTests(unittest.TestCase): surface_document.encode_payload(surface_document.decode_payload(payload)), _vector(detail), file ) elif outcome_kind == "read-only": - self.assertEqual(surface_document.decode_payload(payload), _surface_value(), file) + value, mark = surface_document.decode_payload_marked(payload) + self.assertEqual(value, _surface_value(), file) + self.assertIsNotNone(mark, file) else: with self.assertRaises(ValueError, msg=file) as caught: surface_document.decode_payload(payload) From 09b825d68939b66a633b2ad05654bf970e15f64a Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 04:44:03 +0000 Subject: [PATCH 2/3] Rust: a record with unread slots is held read-only (slot-readonly-rs) (#127) --- packages/cultcache-rs/CHANGELOG.md | 8 + packages/cultcache-rs/README.md | 11 + .../cultcache-rs-derive/src/lib.rs | 18 +- packages/cultcache-rs/src/lib.rs | 252 ++++++++- packages/cultcache-rs/tests/common/mod.rs | 235 +++++++++ packages/cultcache-rs/tests/slot_readonly.rs | 493 ++++++++++++++++++ packages/cultcache-rs/tests/union_parity.rs | 234 +-------- packages/cultmesh-rs/src/lib.rs | 53 ++ packages/cultnet-rs/src/replication.rs | 1 + 9 files changed, 1063 insertions(+), 242 deletions(-) create mode 100644 packages/cultcache-rs/tests/common/mod.rs create mode 100644 packages/cultcache-rs/tests/slot_readonly.rs diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index ed7cd60a..353facea 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -8,6 +8,13 @@ only in git history. ### Breaking +Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0). + +- A record whose payload carries unread slots is read-only. A payload with a slot past its type's declared slots, at any depth (record, nested record, list element, union arm, unit arm), still decodes, and every write or removal of its key is now refused before any store is touched: `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` (the whole batch) and `delete`, so `update` too. The refusal is `SchemaConflictError` with the new `unread_slots`; its message names the type and both slot counts and never a slot value. Holders of such a record used to overwrite it and drop the extra slots silently. The payload nesting refusal still runs first. A held payload this build cannot read at all (an arm it lacks from a newer writer, or a corrupt record) is read-only too: every write or removal is refused with an error naming the type and key (a newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, as no cache call removes it), and `is_writable` answers false. `load_soa` is a write of every held key it omits or changes, so it refuses to drop or replace a read-only record. `is_writable` also answers false for a key a store carries as a foreign record and for a type other than the one holding the key, as the writes do. Each write over a held key decodes the held payload once to ask (about 40% of a `put` of a 200,000-element record, measured in release); keys nothing holds pay nothing. +- cultmesh-rs `pull_rudp_catalog_snapshot` applies documents one by one with `put_raw_envelope`, so a snapshot document for a key the node holds read-only ends the pull at that document and the documents before it stay applied. C# applies a raw snapshot as one cache commit, so there a refusal applies nothing: this is a divergence, owed by follow-up `mesh-snapshot-atomic` (stage the pull through `put_prepared_batch`). +- `SchemaConflictError` gains the public field `unread_slots: Option`; code that builds it with a struct literal must set it (`None`). +- A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it. +- The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check. - A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). - `#[cultcache(index)]` means unique, as in C# `RefuseIndexSharing`: within one type and one alias no two records hold one non-blank value. A put, batch, push or load that would make two records hold a value is refused, naming both keys and the alias, and a store that already holds a duplicate does not open. A grouping index is no longer expressible; a consumer that relied on shared values must drop the `index` and look up by another way. @@ -22,6 +29,7 @@ only in git history. ### Added +- `UnreadSlots { type_name, slot_count, declared_count, occurrences }`, `decode_payload_marked` (the decode that returns the mark) and `CultCache::is_writable(type, key)`. The derives' slot visitors note into a thread-local scope the decode entry opens (`__enter_slots`, `__note_unread_slots`, `__UnitArmSlots`, doc-hidden); a decode outside that entry notes nothing. - `CultCache::foreign_records()` and `ForeignRecord { key, schema_id, schema_name, stored_at }`: the records the last pull found that no registered type claims, sorted by key. - Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name find the one with the smallest key (an index value is unique, below). A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`). - `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::()`. diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 8a8b8f5e..ed9701d5 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -236,6 +236,17 @@ assert_eq!(rmp_serde::from_slice::(&bytes)?, tree); # Ok::<(), Box>(()) ``` +## Records written by a newer build + +A payload that carries a slot past the slots its type declares, in a record, a nested record, a list element or a union +arm (a unit arm declares none), still decodes: the extra slots are skipped. The cache then holds that record read-only, +because writing it would drop what the newer writer stored. `put`, `put_envelope`, `put_raw_envelope`, +`put_prepared_batch` and `delete` of its key are refused with `SchemaConflictError`, whose `unread_slots` names the type +and the slot counts and never a value; `CultCache::is_writable(type, key)` answers the same check. An untouched +read-only record keeps its bytes through every other write. A short payload (missing trailing slots) is not marked, and +a payload nested past `MAX_PAYLOAD_DEPTH` is refused before anything is read. `decode_payload_marked` is the decode that +returns the `UnreadSlots` mark. This runtime has no compatible schema ids, so no mark is consented. + ## Structure Of Arrays The C# CultCache hot path exposes CPU-local SoA scans as: diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index c7f9a3a2..b426ae01 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -381,6 +381,7 @@ fn slot_codec(ident: &syn::Ident, slots: &[CultCacheSlot]) -> proc_macro2::Token where __A: ::serde::de::SeqAccess<'de>, { + let __ticket = ::cultcache_rs::__enter_slots(); #(#option_declarations)* let mut index = 0usize; while index <= #max_slot { @@ -390,7 +391,15 @@ fn slot_codec(ident: &syn::Ident, slots: &[CultCacheSlot]) -> proc_macro2::Token } index += 1; } - while let ::std::option::Option::Some(_) = seq.next_element::<::serde::de::IgnoredAny>()? {} + // The slots past the declared ones are skipped and counted: the decode entry marks the record + // that carried them read-only. + let mut __extra = 0usize; + while seq.next_element::<::serde::de::IgnoredAny>()?.is_some() { + __extra += 1; + } + if __extra > 0 { + ::cultcache_rs::__note_unread_slots(__ticket, stringify!(#ident), #max_slot + 1 + __extra, #max_slot + 1); + } Ok(#ident { #(#build_fields),* }) @@ -567,10 +576,13 @@ fn union_codec(input: DeriveInput) -> syn::Result { quote! { #key => { match seq - .next_element::<::std::option::Option<[(); 0]>>() + .next_element::<::std::option::Option<::cultcache_rs::__UnitArmSlots>>() .map_err(|_| ::serde::de::Error::custom(#arity_message))? { - ::std::option::Option::Some(::std::option::Option::Some(_)) => #ident::#variant, + ::std::option::Option::Some(::std::option::Option::Some(slots)) => { + slots.note(stringify!(#variant)); + #ident::#variant + } ::std::option::Option::Some(::std::option::Option::None) => { return ::std::result::Result::Err(::serde::de::Error::custom( format!(#nil_slots_message, #key), diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 686547f4..59ffa1af 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -800,6 +800,154 @@ pub fn decode_payload(payload: &[u8]) -> Result< decode_grown(payload).map_err(opaque_decode_error) } +/// What a payload decode found past the slots this build declares, at any depth: the first type read with extra slots, +/// how many slots it carried against how many it declares, and how many objects in the payload did. It holds no slot +/// value, so a refusal can name it. A record whose payload carries it is read-only in the cache +/// ([`CultCache::is_writable`]); this runtime has no compatible schema ids, so every mark is unconsented. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UnreadSlots { + /// The first type read with extra slots (a unit arm is named by its variant). + pub type_name: String, + /// The slots that object carried. + pub slot_count: usize, + /// The slots this build declares for it. + pub declared_count: usize, + /// The objects in the payload that carried extra slots. + pub occurrences: usize, +} + +/// The open mark scope of a decode: how many slot objects it has entered, and the mark of the first entered one that +/// carried extra slots (the reference names the first type read, which is the outermost: it is entered before its members). +#[derive(Default)] +struct MarkScope { + entered: usize, + mark: Option<(usize, UnreadSlots)>, +} + +thread_local! { + /// The open mark scope of a decode on this thread: `None` when no decode entry has one open, so a decode through + /// serde alone notes nothing. The slot visitors the derives write note into it. + static UNREAD_SLOTS: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +/// The derives' sensor, first half: a slot object is entered. Its ticket orders it among the objects of the decode. +#[doc(hidden)] +pub fn __enter_slots() -> usize { + UNREAD_SLOTS.with(|scope| { + scope.borrow_mut().as_mut().map_or(0, |open| { + open.entered += 1; + open.entered + }) + }) +} + +/// The derives' sensor, second half: the object entered as `ticket`, of `type_name`, carried `carried` slots where this +/// build declares `declared`. +#[doc(hidden)] +pub fn __note_unread_slots(ticket: usize, type_name: &str, carried: usize, declared: usize) { + UNREAD_SLOTS.with(|scope| { + let mut scope = scope.borrow_mut(); + let Some(open) = scope.as_mut() else { + return; + }; + let occurrences = open.mark.as_ref().map_or(0, |(_, found)| found.occurrences) + 1; + if open.mark.as_ref().is_none_or(|(first, _)| ticket < *first) { + let named = UnreadSlots { type_name: type_name.to_string(), slot_count: carried, declared_count: declared, occurrences }; + open.mark = Some((ticket, named)); + } else if let Some((_, found)) = open.mark.as_mut() { + found.occurrences = occurrences; + } + }); +} + +/// A unit arm is a zero-slot record: its slots array is read and counted, not refused. +#[doc(hidden)] +pub struct __UnitArmSlots { + ticket: usize, + extra: usize, +} + +impl __UnitArmSlots { + /// Notes the arm if its array held any element. + pub fn note(self, variant: &str) { + if self.extra > 0 { + __note_unread_slots(self.ticket, variant, self.extra, 0); + } + } +} + +impl<'de> serde::Deserialize<'de> for __UnitArmSlots { + fn deserialize>(deserializer: D) -> std::result::Result { + struct Count; + impl<'de> serde::de::Visitor<'de> for Count { + type Value = __UnitArmSlots; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a slots array") + } + fn visit_seq>(self, mut seq: A) -> std::result::Result { + let ticket = __enter_slots(); + let mut extra = 0usize; + while seq.next_element::()?.is_some() { + extra += 1; + } + Ok(__UnitArmSlots { ticket, extra }) + } + } + deserializer.deserialize_seq(Count) + } +} + +/// The scope one marked decode opens on this thread; closed when it ends, whatever the decode did. A marked decode is +/// never re-entered (only the cache's own checks open one), so scopes do not nest. +struct UnreadScope; + +impl UnreadScope { + fn open() -> Self { + UNREAD_SLOTS.with(|scope| *scope.borrow_mut() = Some(MarkScope::default())); + Self + } + + fn mark(&self) -> Option { + UNREAD_SLOTS.with(|scope| scope.borrow_mut().as_mut().and_then(|open| open.mark.take()).map(|(_, mark)| mark)) + } +} + +impl Drop for UnreadScope { + fn drop(&mut self) { + UNREAD_SLOTS.with(|scope| *scope.borrow_mut() = None); + } +} + +/// [`decode_payload`] with the mark: the same depth check and grown decode, inside a scope the derives' slot +/// visitors note into. A payload past the nesting limit is refused before the decode, never marked. +pub fn decode_payload_marked(payload: &[u8]) -> Result<(T, Option)> { + check_payload_depth(payload)?; + let scope = UnreadScope::open(); + let value = decode_grown(payload).map_err(opaque_decode_error)?; + Ok((value, scope.mark())) +} + +/// The registered check of a held payload: monomorphized by [`CultCache::register_entry_type`] so the untyped write +/// entries can decode a payload they cannot name. +/// A payload the type cannot decode at all (an arm this build lacks, a corrupt record) is an error here, and the held +/// record is read-only: the cache cannot tell what a writer of another build stored in it. +fn unread_slots_of(payload: &[u8]) -> Result> { + decode_payload_marked::(payload).map(|(_, mark)| mark) +} + +/// What the cache finds in a held payload when asked to write or remove its key. +enum HeldFinding { + Free, + Marked(UnreadSlots), + Unreadable(anyhow::Error), +} + +/// What a registered type is to the cache: its schema name and how to read the slots a held payload carries. +struct Definition { + schema_name: &'static str, + unread_slots: fn(&[u8]) -> Result>, +} + /// The nesting bound for an envelope admitted without a typed decode, from the same walk that /// [`decode_payload`] runs first. Names the entry's type and key, never the payload. fn require_payload_depth(entry: &CultCacheEnvelope) -> Result<()> { @@ -938,10 +1086,26 @@ pub struct SchemaConflictError { pub schema_id: String, pub schema_names: Vec, pub record_key: String, + /// Set when the refusal is a write or removal of a record held read-only for the slots its payload carries past + /// this build's; `None` for every other refusal. + pub unread_slots: Option, } impl std::fmt::Display for SchemaConflictError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(unread) = &self.unread_slots { + return write!( + formatter, + "{} record {} is read-only: {} was read with {} slots where this build declares {} ({} object(s) in the record). \ + Writing it would drop data a newer writer stored; read it with types that declare those slots, or write it from that build.", + self.schema_names.first().map_or("", String::as_str), + self.record_key, + unread.type_name, + unread.slot_count, + unread.declared_count, + unread.occurrences + ); + } write!( formatter, "schema id {:?} cannot describe record {:?}: {:?}", @@ -2738,7 +2902,7 @@ struct CultCacheStoreRegistration { } pub struct CultCache { - definitions: BTreeMap, + definitions: BTreeMap, /// The registered types declared `#[cultcache(global)]`. globals: BTreeSet, schema_name_definitions: BTreeMap, @@ -2825,8 +2989,8 @@ impl CultCache { "CultCache entry types must declare a non-empty type" )); } - if let Some(existing_schema) = self.definitions.get(T::TYPE) - && *existing_schema != T::SCHEMA_NAME + if let Some(existing) = self.definitions.get(T::TYPE) + && existing.schema_name != T::SCHEMA_NAME { return Err(anyhow!( "CultCache already has a different definition registered for type {:?}", @@ -2875,7 +3039,7 @@ impl CultCache { } } } - self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME); + self.definitions.insert(T::TYPE.to_string(), Definition { schema_name: T::SCHEMA_NAME, unread_slots: unread_slots_of:: }); self.schema_name_definitions .insert(T::SCHEMA_NAME.to_string(), T::TYPE.to_string()); Ok(()) @@ -3036,6 +3200,60 @@ impl CultCache { Ok(Some(payload)) } + /// Whether a write or removal of `(type_id, key)` would be admitted: false when the held payload carries slots past + /// its registered type's declared slots, at any depth, or cannot be read by this build at all (the record is + /// read-only), when a store carries a foreign record at the key, or when another registered type holds it. True for a + /// key nothing holds. An unregistered type is an error. Advisory: the write entries ask the same checks themselves. + pub fn is_writable(&self, type_id: &str, key: &str) -> Result { + if !matches!(self.held_finding(type_id, key)?, HeldFinding::Free) { + return Ok(false); + } + let probe = CultCacheEnvelope { + key: key.to_string(), + r#type: type_id.to_string(), + payload: Vec::new(), + stored_at: String::new(), + schema_id: None, + }; + Ok(self.home_index(type_id).is_ok() && self.refuse_retype(&self.entries, &self.foreign, &probe).is_ok()) + } + + /// The one decode behind every write entry and [`CultCache::is_writable`]: the held envelope at `(type_id, key)` + /// read under the registered type's mark. + fn held_finding(&self, type_id: &str, key: &str) -> Result { + let Some(definition) = self.definitions.get(type_id) else { + bail!("No entry type registered for CultCache envelope type {type_id:?}"); + }; + let Some(held) = self.entries.get(&entry_id_parts(type_id, key)) else { + return Ok(HeldFinding::Free); + }; + Ok(match (definition.unread_slots)(&held.payload) { + Ok(None) => HeldFinding::Free, + Ok(Some(unread)) => HeldFinding::Marked(unread), + Err(error) => HeldFinding::Unreadable(error), + }) + } + + /// Refuses a write or removal of a record held read-only, before any store is touched: for the slots its payload + /// carries past this build's, or because this build cannot read the payload at all. + fn refuse_unread_slots(&self, type_id: &str, key: &str) -> Result<()> { + match self.held_finding(type_id, key)? { + HeldFinding::Free => Ok(()), + HeldFinding::Unreadable(error) => Err(error.context(format!( + "CultCache entry {type_id:?} at key {key:?} is held read-only: this build cannot read its payload. A newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, not through the cache" + ))), + HeldFinding::Marked(unread) => { + let held = &self.entries[&entry_id_parts(type_id, key)]; + Err(anyhow::Error::new(SchemaConflictError { + schema_id: schema_identity(held).to_string(), + schema_names: vec![self.definitions[type_id].schema_name.to_string()], + record_key: key.to_string(), + unread_slots: Some(unread), + })) + } + } + } + pub fn get_required(&self, key: &str) -> Result { self.get::(key)? .ok_or_else(|| anyhow!("CultCache has no {:?} entry at key {:?}", T::TYPE, key)) @@ -3166,6 +3384,7 @@ impl CultCache { pub fn put(&mut self, key: impl Into, value: &T) -> Result { let (entry, parsed) = self.prepare_entry(key, value)?; + self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; self.refuse_retype(&self.entries, &self.foreign, &entry)?; let change = self.judge_unique(&[&entry])?; @@ -3266,6 +3485,7 @@ impl CultCache { )); } require_payload_depth(entry)?; + self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_claimed_id(entry)?; let Some(home) = self.home_index(&entry.r#type)? else { continue; @@ -3308,6 +3528,7 @@ impl CultCache { pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { let parsed = self.validate_envelope::(&entry)?; + self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_claimed_id(&entry)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; self.refuse_retype(&self.entries, &self.foreign, &entry)?; @@ -3345,6 +3566,7 @@ impl CultCache { )); } require_payload_depth(&entry)?; + self.refuse_unread_slots(&entry.r#type, &entry.key)?; self.refuse_claimed_id(&entry)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; self.refuse_retype(&self.entries, &self.foreign, &entry)?; @@ -3400,6 +3622,7 @@ impl CultCache { pub fn delete(&mut self, key: &str) -> Result { self.require_entry_type::()?; + self.refuse_unread_slots(T::TYPE, key)?; let id = entry_id_parts(T::TYPE, key); let Some(entry) = self.entries.get(&id).cloned() else { if let Some((_, foreign)) = self.foreign.iter().find(|(_, foreign)| foreign.key == key) { @@ -3427,7 +3650,9 @@ impl CultCache { /// Replaces the cache's view with the table's records. Every record is /// checked (registered type, and a home store once any store is attached) /// into a local image first, so a refused load admits nothing and leaves - /// the prior view intact. Loading writes no store. + /// the prior view intact. Loading writes no store, but a held record that is read-only (unread slots, or a payload + /// this build cannot read) may not be omitted or replaced by the table: the next batch would lay the image over the + /// store. pub fn load_soa(&mut self, table: CultCacheSoaTable) -> Result<()> { let mut loaded = BTreeMap::new(); for entry in table.into_envelopes()? { @@ -3442,6 +3667,12 @@ impl CultCache { self.refuse_retype(&loaded, &self.foreign, &entry)?; loaded.insert(entry_id(&entry), entry); } + // A load is a write of every held key it omits or changes: the one check refuses those held read-only. + for ((type_id, key), held) in &self.entries { + if loaded.get(&(type_id.clone(), key.clone())).is_none_or(|incoming| incoming.payload != held.payload) { + self.refuse_unread_slots(type_id, key)?; + } + } self.refuse_second_globals(&loaded)?; let holders = self.judge_image(&loaded)?; self.entries = loaded; @@ -3518,7 +3749,7 @@ impl CultCache { } for (slot, keys) in &ordered { if let [first, second, ..] = keys.as_slice() { - let schema = self.definitions.get(&slot.0).copied().unwrap_or(&slot.0); + let schema = self.definitions.get(&slot.0).map_or(slot.0.as_str(), |definition| definition.schema_name); let violation = [slot.1.as_str(), first.as_str(), second.as_str(), schema]; if least.is_none_or(|best| violation_cmp(&violation, &best).is_lt()) { least = Some(violation); @@ -3553,7 +3784,7 @@ impl CultCache { .find(|held_key| held_key.as_str() != key); match held_other { Some(existing_key) => Err(CultCacheSecondGlobal { - schema_name: self.definitions.get(type_id).copied().unwrap_or(type_id).to_string(), + schema_name: self.definitions.get(type_id).map_or(type_id, |definition| definition.schema_name).to_string(), existing_key: existing_key.clone(), key: key.to_string(), } @@ -3597,6 +3828,7 @@ impl CultCache { schema_id: schema_identity(entry).to_string(), schema_names: vec![resolved, entry.r#type.clone()], record_key: entry.key.clone(), + unread_slots: None, })), _ => Ok(()), } @@ -3611,7 +3843,7 @@ impl CultCache { fn require_entry_type(&self) -> Result<()> { match self.definitions.get(T::TYPE) { - Some(schema_name) if *schema_name == T::SCHEMA_NAME => Ok(()), + Some(definition) if definition.schema_name == T::SCHEMA_NAME => Ok(()), _ => Err(anyhow!( "CultCache entry type {:?} is not registered on this cache instance", T::TYPE @@ -3693,6 +3925,7 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope], disk: &DiskStore) -> Res schema_id, schema_names: vec![existing_type, entry.r#type.clone()], record_key: entry.key.clone(), + unread_slots: None, })); } } @@ -3710,6 +3943,7 @@ fn encode_store_snapshot(entries: &[CultCacheEnvelope], disk: &DiskStore) -> Res schema_id, schema_names: vec![arrived_type, document_type], record_key: key, + unread_slots: None, })); } } @@ -3846,6 +4080,7 @@ fn foreign_conflict(carried: &CultCacheEnvelope, written_type: &str) -> anyhow:: schema_id: schema_identity(carried).to_string(), schema_names: vec![carried.r#type.clone(), written_type.to_string()], record_key: carried.key.clone(), + unread_slots: None, }) } @@ -3856,6 +4091,7 @@ fn retype_conflict(held: &CultCacheEnvelope, written: &CultCacheEnvelope) -> any schema_id: schema_identity(written).to_string(), schema_names: vec![held.r#type.clone(), written.r#type.clone()], record_key: written.key.clone(), + unread_slots: None, }) } diff --git a/packages/cultcache-rs/tests/common/mod.rs b/packages/cultcache-rs/tests/common/mod.rs new file mode 100644 index 00000000..3f893c0f --- /dev/null +++ b/packages/cultcache-rs/tests/common/mod.rs @@ -0,0 +1,235 @@ +//! The union-parity vectors' types and constructed value, shared by the test files that walk them. +#![allow(dead_code)] + +use std::path::PathBuf; + +use cultcache_rs::CultSlots; +use cultcache_rs::CultUnion; +use cultcache_rs::DatabaseEntry; + + +#[derive(Clone, Debug, PartialEq, CultUnion)] +pub enum Node { + #[cultcache(key = 0)] + Box(BoxArm), + #[cultcache(key = 1)] + Text(TextArm), + #[cultcache(key = 2)] + Use(UseArm), + #[cultcache(key = 5)] + Image(ImageArm), + #[cultcache(key = 6)] + Wrap(WrapArm), +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct WrapArm { + #[cultcache(key = 0)] + pub inner: Option>, + #[cultcache(key = 1)] + pub id: String, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct BoxArm { + #[cultcache(key = 0)] + pub id: String, + #[cultcache(key = 1)] + pub children: Vec, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct TextArm { + #[cultcache(key = 0)] + pub id: String, + #[cultcache(key = 1)] + pub content: String, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct UseArm { + #[cultcache(key = 0)] + pub id: String, + #[cultcache(key = 1)] + pub composition: Composition, + #[cultcache(key = 2)] + pub slots: Vec>, +} + +// Slot 2 is retired: a gap written nil. +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct ImageArm { + #[cultcache(key = 0)] + pub id: String, + #[cultcache(key = 1)] + pub uri: String, + #[cultcache(key = 3)] + pub alt: String, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct Composition { + #[cultcache(key = 0)] + pub name: String, + #[cultcache(key = 1)] + pub body: Option>, +} + +#[derive(Clone, Debug, PartialEq, CultUnion)] +pub enum Outcome { + #[cultcache(key = 0)] + Pending(PendingArm), + #[cultcache(key = 1)] + Completed(CompletedArm), + #[cultcache(key = 2)] + Denied(IdArm), + #[cultcache(key = -2147483648)] + Floor(IdArm), + #[cultcache(key = 2147483647)] + Ceiling(IdArm), +} + +// The codec's unit arm (no members, written [key, []]), kept apart from the golden: a document list's arms carry ids. +#[derive(Clone, Debug, PartialEq, CultUnion)] +pub enum UnitOutcome { + #[cultcache(key = 2)] + Denied, + #[cultcache(key = -2147483648)] + Floor, + #[cultcache(key = 2147483647)] + Ceiling, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct PendingArm { + #[cultcache(key = 0)] + pub message: String, + #[cultcache(key = 1)] + pub id: String, +} + +// An arm with no members of its own: just its element id. +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct IdArm { + #[cultcache(key = 0)] + pub id: String, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct CompletedArm { + #[cultcache(key = 0)] + pub source_version: i32, + #[cultcache(key = 1)] + pub id: String, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct NamedArm { + #[cultcache(key = 0)] + pub name: String, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct UsesArm { + #[cultcache(key = 0)] + pub name: String, + #[cultcache(key = 1)] + pub uses: i32, +} + +#[derive(Clone, Debug, PartialEq, CultSlots)] +pub struct CountArm { + #[cultcache(key = 0)] + pub name: String, + #[cultcache(key = 1)] + pub count: i32, +} + +#[derive(Clone, Debug, PartialEq, CultUnion)] +pub enum Item { + #[cultcache(key = 0)] + Gem(NamedArm), + #[cultcache(key = 1)] + Tool(UsesArm), + #[cultcache(key = 2)] + Part(CountArm), +} + +// The same arms as Item under its own keys: key 2 is unknown to it. +#[derive(Clone, Debug, PartialEq, CultUnion)] +pub enum CraftedItem { + #[cultcache(key = 0)] + Tool(UsesArm), + #[cultcache(key = 1)] + Part(CountArm), +} + +#[derive(Clone, Debug, PartialEq, DatabaseEntry)] +#[cultcache(type = "tests.union_surface", schema = "tests.union_surface.v1")] +pub struct UnionSurface { + #[cultcache(key = 0)] + pub id: String, + #[cultcache(key = 1)] + pub version: i32, + #[cultcache(key = 2)] + pub root: Option, + #[cultcache(key = 3)] + pub outcomes: Vec, + #[cultcache(key = 4)] + pub part: Option, + #[cultcache(key = 5)] + pub crafted: Option, + #[cultcache(key = 6)] + pub missing: Option, +} + +pub fn text(id: &str, content: &str) -> Node { + Node::Text(TextArm { id: id.into(), content: content.into() }) +} + +pub fn image(id: &str, uri: &str, alt: &str) -> Node { + Node::Image(ImageArm { id: id.into(), uri: uri.into(), alt: alt.into() }) +} + +pub fn boxed(id: &str, children: Vec) -> Node { + Node::Box(BoxArm { id: id.into(), children }) +} + +pub fn constructed() -> UnionSurface { + let inner = boxed("0000000000d0", vec![text("0000000000d1", "x"), image("0000000000d2", "uri://a", "alt")]); + UnionSurface { + id: "surface".into(), + version: 3, + root: Some(boxed( + "0000000000c0", + vec![ + text("0000000000c1", "hello"), + Node::Use(UseArm { + id: "0000000000c2".into(), + composition: Composition { name: "comp".into(), body: Some(std::boxed::Box::new(inner)) }, + slots: vec![Some(text("0000000000e1", "slot")), None, Some(boxed("0000000000e2", vec![]))], + }), + image("0000000000c3", "uri://b", ""), + ], + )), + outcomes: vec![ + Outcome::Pending(PendingArm { message: "wait".into(), id: "0000000000a1".into() }), + Outcome::Completed(CompletedArm { source_version: 7, id: "0000000000a2".into() }), + Outcome::Denied(id_arm("0000000000a3")), + ], + part: Some(Item::Tool(UsesArm { name: "hammer".into(), uses: 12 })), + crafted: Some(CraftedItem::Part(CountArm { name: "gear".into(), count: 4 })), + missing: None, + } +} + +pub fn id_arm(id: &str) -> IdArm { + IdArm { id: id.into() } +} + +pub fn vector(name: &str) -> Vec { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/vectors/union-parity") + .join(name); + std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display())) +} diff --git a/packages/cultcache-rs/tests/slot_readonly.rs b/packages/cultcache-rs/tests/slot_readonly.rs new file mode 100644 index 00000000..5bffcd9a --- /dev/null +++ b/packages/cultcache-rs/tests/slot_readonly.rs @@ -0,0 +1,493 @@ +//! A record whose payload carries unread slots at any depth is read-only: the C# reference's union-parity vectors +//! (README "Extra trailing slot" and "Slot stores") walked through the cache. This runtime has no compatible schema +//! ids, so every mark is unconsented and no `explained` outcome can arise. + +mod common; + +use std::path::Path; + +use common::*; +use cultcache_rs::CultCache; +use cultcache_rs::CultCacheEnvelope; +use cultcache_rs::DatabaseEntry; +use cultcache_rs::SchemaConflictError; +use cultcache_rs::SingleFileMessagePackBackingStore; +use cultcache_rs::UnreadSlots; +use cultcache_rs::decode_payload_marked; + +/// A row of `slot-outcomes.msgpack`: store, key, writable, explained, type, slots, declared, occurrences, flattenRefusedBy. +type SlotRow = (String, String, bool, Option, Option, Option, Option, Option, Option); + +const SLOT_VALUE: &str = "extra-slot"; + +/// `tests.slot_unit`, a document `[Name, Outcome: UnitOutcome]`. +#[derive(Clone, Debug, PartialEq, DatabaseEntry)] +#[cultcache(type = "tests.slot_unit", schema = "tests.slot_unit.v1")] +struct SlotUnit { + #[cultcache(key = 0)] + name: String, + #[cultcache(key = 1)] + outcome: UnitOutcome, +} + +/// Two declared slots, the second defaulted: a payload of one slot is short, not marked. +#[derive(Clone, Debug, PartialEq, DatabaseEntry)] +#[cultcache(type = "tests.shortened", schema = "tests.shortened.v1")] +struct Shortened { + #[cultcache(key = 0)] + id: String, + #[cultcache(key = 1, default)] + note: String, +} + +fn registered() -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache +} + +/// Any MessagePack value a store holds, enough to drop one record from a store and write the rest back. +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +enum Packed { + Nil(()), + Bool(bool), + Int(i64), + Text(String), + Bin(#[serde(with = "serde_bytes")] Vec), + Array(Vec), +} + +/// The committed store without its `marked-variant` record. A variant is a five-slot record, which this runtime's +/// store reader refuses whole (`the_variant_row_...`), so the other rows are walked from a copy without it. +fn without_variant_row(store: &[u8]) -> Vec { + let Packed::Array(mut parts) = rmp_serde::from_slice(store).unwrap() else { panic!("a store is an array") }; + let Packed::Array(records) = parts.last_mut().unwrap() else { panic!("a store ends in its records") }; + let before = records.len(); + records.retain(|record| !matches!(record, Packed::Array(fields) if matches!(fields.first(), Some(Packed::Text(key)) if key == "marked-variant"))); + assert_eq!(records.len(), before - 1); + rmp_serde::to_vec(&parts).unwrap() +} + +/// A copy of a committed store, opened with both types registered; the vector itself is never written. The slot store +/// is read without its variant row. +fn open_copy(file: &str, dir: &Path) -> (CultCache, std::path::PathBuf) { + let path = dir.join(file); + let bytes = if file == "extra-slot-store.cc" { without_variant_row(&vector(file)) } else { vector(file) }; + std::fs::write(&path, bytes).unwrap(); + let mut cache = registered(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); + (cache, path) +} + +fn envelope(key: &str, payload: Vec) -> CultCacheEnvelope { + CultCacheEnvelope { + key: key.into(), + r#type: UnionSurface::TYPE.into(), + payload, + stored_at: "2026-01-01T00:00:00Z".into(), + schema_id: None, + } +} + +fn conflict(error: &anyhow::Error) -> &SchemaConflictError { + error.downcast_ref::().unwrap_or_else(|| panic!("not a schema conflict: {error:#}")) +} + +fn mark(type_name: &str, slot_count: usize, declared_count: usize, occurrences: usize) -> UnreadSlots { + UnreadSlots { type_name: type_name.into(), slot_count, declared_count, occurrences } +} + +/// The marks the README table pins, by `detail`. The reference names an arm by its class; here an arm is the +/// `...Arm` struct the union wraps. +fn expected_mark(detail: &str) -> UnreadSlots { + match detail { + "record" | "depth" => mark("UnionSurface", 8, 7, 1), + "nested-record" => mark("Composition", 3, 2, 1), + "arm" => mark("BoxArm", 3, 2, 1), + "list-element" => mark("TextArm", 3, 2, 1), + other => panic!("unknown detail {other}"), + } +} + +fn assert_refused_with(error: &anyhow::Error, key: &str, expected: &UnreadSlots) { + let refusal = conflict(error); + assert_eq!(refusal.unread_slots.as_ref(), Some(expected)); + assert_eq!(refusal.record_key, key); + assert_eq!(refusal.schema_names.len(), 1); + let message = error.to_string(); + assert!(message.contains(&expected.type_name), "{message}"); + assert!(message.contains(&expected.slot_count.to_string()) && message.contains(&expected.declared_count.to_string()), "{message}"); + assert!(!message.contains(SLOT_VALUE), "a refusal echoed the slot value: {message}"); +} + +#[test] +fn every_read_only_vector_decodes_marked_and_is_not_writable() { + let rows: Vec<(String, String, String, Option, String)> = rmp_serde::from_slice(&vector("outcomes.msgpack")).unwrap(); + assert_eq!(rows.len(), 9); + let bounds: Vec = rmp_serde::from_slice(&vector("bounds-outcomes.msgpack")).unwrap(); + let mut cases: Vec<(String, String)> = rows + .into_iter() + .filter(|(_, outcome, ..)| outcome == "read-only") + .map(|(file, _, _, _, detail)| (file, detail)) + .collect(); + for entry in &bounds[1..] { + let fields: Vec<&str> = entry.as_array().unwrap().iter().filter_map(|field| field.as_str()).collect(); + if fields[1] == "read-only" { + cases.push((fields[0].to_string(), fields[3].to_string())); + } + } + assert_eq!(cases.len(), 5); + for (file, detail) in cases { + let payload = vector(&file); + let (value, found) = decode_payload_marked::(&payload).unwrap_or_else(|error| panic!("{file}: {error:#}")); + assert_eq!(value, constructed(), "{file}"); + assert_eq!(found, Some(expected_mark(&detail)), "{file}"); + + let mut cache = registered(); + let dir = tempfile::tempdir().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc"))).unwrap(); + cache.put_raw_envelope(envelope("k", payload.clone())).unwrap(); + assert_eq!(cache.get::("k").unwrap(), Some(constructed()), "{file}"); + assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap(), "{file}"); + assert!(cache.is_writable(UnionSurface::TYPE, "unheld").unwrap(), "{file}"); + let error = cache.put("k", &constructed()).unwrap_err(); + assert_refused_with(&error, "k", &expected_mark(&detail)); + } +} + +#[test] +fn clean_and_short_payloads_are_writable_and_unmarked() { + let (value, found) = decode_payload_marked::(&vector("union-surface.msgpack")).unwrap(); + assert_eq!((value, found), (constructed(), None)); + // A payload short of its declared slots keeps its defaults and does not mark. + let (value, found) = decode_payload_marked::(&rmp_serde::to_vec(&("only",)).unwrap()).unwrap(); + assert_eq!((value.id.as_str(), value.note.as_str(), found), ("only", "", None)); +} + +#[test] +fn a_payload_past_the_nesting_limit_is_refused_never_marked() { + for file in ["depth-skipped-past-limit.msgpack", "depth-past-limit.msgpack"] { + let error = decode_payload_marked::(&vector(file)).unwrap_err(); + assert!(error.to_string().contains("the limit is 256"), "{file}: {error:#}"); + } +} + +#[test] +fn a_plain_serde_decode_marks_nothing_and_does_not_leak_a_scope() { + // No scope is open: the visitors note into nothing. + let value: UnionSurface = rmp_serde::from_slice(&vector("extra-slot-record.msgpack")).unwrap(); + assert_eq!(value, constructed()); + // A marked decode followed by a clean one: the first mark does not survive into the second. + assert!(decode_payload_marked::(&vector("extra-slot-record.msgpack")).unwrap().1.is_some()); + assert_eq!(decode_payload_marked::(&vector("union-surface.msgpack")).unwrap().1, None); +} + +#[test] +fn the_slot_store_rows_walk_through_the_cache() { + let rows: Vec = + rmp_serde::from_slice(&vector("slot-outcomes.msgpack")).unwrap(); + assert_eq!(rows.len(), 8); + let dir = tempfile::tempdir().unwrap(); + let mut walked = 0; + for (store, key, writable, _explained, type_name, slots, declared, occurrences, flatten_refused_by) in rows { + // The variant row is a record this runtime cannot hold: see the variant test. The consent store's `writable` and + // `explained` answer the compatible-id declaration, which this runtime has none of; its marks are walked below. + if flatten_refused_by.is_some() || store == "extra-slot-consent.cc" { + continue; + } + let case = dir.path().join(&key); + std::fs::create_dir_all(&case).unwrap(); + let (mut cache, _) = open_copy(&store, &case); + let type_id = if key == "unit-arm" { SlotUnit::TYPE } else { UnionSurface::TYPE }; + assert_eq!(cache.is_writable(type_id, &key).unwrap(), writable, "{store} {key}"); + let removal = if key == "unit-arm" { cache.delete::(&key) } else { cache.delete::(&key) }; + match writable { + true => assert!(removal.unwrap()), + false => { + let found = conflict(&removal.unwrap_err()).unread_slots.clone().unwrap(); + // The reference names the unit arm's class, Denied2; here the variant. + let expected_type = match type_name.as_deref().unwrap() { + "Denied2" => "Denied", + other => other, + }; + assert_eq!((found.type_name.as_str(), Some(found.slot_count), Some(found.declared_count), Some(found.occurrences)), (expected_type, slots, declared, occurrences)); + assert!(cache.snapshot_envelopes().iter().any(|held| held.key == key), "a refused removal removed {key}"); + } + } + walked += 1; + } + assert_eq!(walked, 3); +} + +#[test] +fn the_consent_store_marks_are_unconsented_here() { + let rows: Vec = + rmp_serde::from_slice(&vector("slot-outcomes.msgpack")).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut found = 0; + for (store, key, _writable, _explained, _type_name, slots, declared, occurrences, _) in rows { + if store != "extra-slot-consent.cc" { + continue; + } + let (cache, _) = open_copy(&store, dir.path()); + let held = cache.snapshot_envelopes().into_iter().find(|held| held.key == key).unwrap(); + let (_, mark) = decode_payload_marked::(&held.payload).unwrap(); + match slots { + // The reference's `consented` row is writable there (a listed compatible id covers it); here every mark refuses. + None => assert_eq!(mark, None, "{key}"), + Some(slots) => { + let mark = mark.unwrap_or_else(|| panic!("{key} carried no mark")); + // The first type read is the document, whose reference name is SurfaceDeclaring. + assert_eq!(mark.type_name, "UnionSurface", "{key}"); + assert_eq!((Some(mark.slot_count), Some(mark.declared_count), Some(mark.occurrences)), (Some(slots), declared, occurrences), "{key}"); + assert!(!cache.is_writable(UnionSurface::TYPE, &key).unwrap(), "{key}"); + } + } + found += 1; + } + assert_eq!(found, 4); +} + +#[test] +fn every_write_entry_refuses_a_marked_key_before_any_store_is_touched() { + let dir = tempfile::tempdir().unwrap(); + let (mut cache, path) = open_copy("extra-slot-store.cc", dir.path()); + let expected = mark("Composition", 3, 2, 1); + let before = std::fs::read(&path).unwrap(); + let held = cache.snapshot_envelopes().into_iter().find(|held| held.key == "marked").unwrap(); + let clean = cache.snapshot_envelopes().into_iter().find(|held| held.key == "clean").unwrap(); + assert!(!cache.is_writable(UnionSurface::TYPE, "marked").unwrap()); + assert!(cache.is_writable(UnionSurface::TYPE, "clean").unwrap()); + + let replacing = envelope("marked", rmp_serde::to_vec(&constructed()).unwrap()); + assert_refused_with(&cache.put("marked", &constructed()).unwrap_err(), "marked", &expected); + assert_refused_with(&cache.put_envelope::(replacing.clone()).unwrap_err(), "marked", &expected); + assert_refused_with(&cache.put_raw_envelope(replacing.clone()).unwrap_err(), "marked", &expected); + assert_refused_with(&cache.put_prepared_batch(vec![replacing]).unwrap_err(), "marked", &expected); + assert_refused_with(&cache.delete::("marked").unwrap_err(), "marked", &expected); + assert_refused_with( + &cache.update::("marked", |current| current.unwrap()).unwrap_err(), + "marked", + &expected, + ); + + // A batch with a marked key refuses whole: its clean member is not written either. + let mut rewritten = clean.clone(); + rewritten.payload = rmp_serde::to_vec(&UnionSurface { version: 9, ..constructed() }).unwrap(); + let batch = vec![rewritten, envelope("marked", rmp_serde::to_vec(&constructed()).unwrap())]; + assert_refused_with(&cache.put_prepared_batch(batch).unwrap_err(), "marked", &expected); + + assert_eq!(std::fs::read(&path).unwrap(), before, "a refused write touched the store"); + assert_eq!(cache.snapshot_envelopes().into_iter().find(|entry| entry.key == "marked").unwrap(), held); + assert_eq!(cache.snapshot_envelopes().into_iter().find(|entry| entry.key == "clean").unwrap(), clean); + + // A write to another key succeeds, and the marked record's bytes, schema id and stored-at are carried unchanged. + cache.put("clean", &UnionSurface { version: 9, ..constructed() }).unwrap(); + cache.put("other", &constructed()).unwrap(); + assert!(cache.delete::("other").unwrap()); + let mut reopened = registered(); + reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + reopened.pull_all_backing_stores().unwrap(); + let carried = reopened.snapshot_envelopes().into_iter().find(|entry| entry.key == "marked").unwrap(); + assert_eq!(carried, held); + assert_eq!(carried.payload, vector("extra-slot-nested-record.msgpack")); + assert_eq!(reopened.get::("clean").unwrap().unwrap().version, 9); +} + +#[test] +fn a_unit_arm_with_slots_decodes_as_the_unit_arm_and_marks() { + let (document, found) = decode_payload_marked::(&unit_arm_payload()).unwrap(); + assert_eq!(document, SlotUnit { name: "unit-arm".into(), outcome: UnitOutcome::Denied }); + assert_eq!(found, Some(mark("Denied", 1, 0, 1))); + + let dir = tempfile::tempdir().unwrap(); + let (mut cache, _) = open_copy("extra-slot-store.cc", dir.path()); + assert_eq!(cache.get::("unit-arm").unwrap(), Some(document.clone())); + assert!(!cache.is_writable(SlotUnit::TYPE, "unit-arm").unwrap()); + let error = cache.put("unit-arm", &document).unwrap_err(); + assert_refused_with(&error, "unit-arm", &mark("Denied", 1, 0, 1)); + assert_refused_with(&cache.delete::("unit-arm").unwrap_err(), "unit-arm", &mark("Denied", 1, 0, 1)); + + // Several elements are still one marked object, and a clean unit arm is unmarked. + let (_, found) = decode_payload_marked::(&rmp_serde::to_vec(&(2u64, ("a", "b", "c"))).unwrap()).unwrap(); + assert_eq!(found, Some(mark("Denied", 3, 0, 1))); + let (_, found) = decode_payload_marked::(&vector("unit-arm-denied.msgpack")).unwrap(); + assert_eq!(found, None); +} + +fn unit_arm_payload() -> Vec { + let dir = tempfile::tempdir().unwrap(); + let (cache, _) = open_copy("extra-slot-store.cc", dir.path()); + cache.snapshot_envelopes().into_iter().find(|held| held.key == "unit-arm").unwrap().payload +} + +#[test] +fn the_variant_row_of_the_slot_store_is_refused_whole_by_the_store_reader() { + // `marked-variant` is a five-slot record (a variant): this runtime holds no variants, and its v1 snapshot reader + // refuses such a record, so the store as written cannot load here and the other rows are walked without it. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("extra-slot-store.cc"); + std::fs::write(&path, vector("extra-slot-store.cc")).unwrap(); + let mut cache = registered(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + let error = cache.pull_all_backing_stores().unwrap_err(); + assert!(format!("{error:#}").contains("is not decodable"), "{error:#}"); +} + +fn slot_unit_envelope(key: &str, payload: Vec) -> CultCacheEnvelope { + CultCacheEnvelope { r#type: SlotUnit::TYPE.into(), ..envelope(key, payload) } +} + +#[test] +fn a_held_payload_the_type_cannot_read_is_read_only_and_refused_by_name() { + let mut cache = registered(); + // A newer writer's record: an arm this build lacks and an extra root slot. get cannot read it, and it is the data + // read-only exists to protect. + let newer = rmp_serde::to_vec(&("CANARY-name", (99u64, ("x",)), "CANARY-extra")).unwrap(); + cache.put_raw_envelope(slot_unit_envelope("n", newer)).unwrap(); + assert!(cache.get::("n").is_err()); + assert!(!cache.is_writable(SlotUnit::TYPE, "n").unwrap()); + let replacement = SlotUnit { name: "n".into(), outcome: UnitOutcome::Denied }; + let held = cache.snapshot_envelopes(); + for error in [ + cache.put("n", &replacement).unwrap_err(), + cache.put_raw_envelope(slot_unit_envelope("n", rmp_serde::to_vec(&replacement).unwrap())).unwrap_err(), + cache.delete::("n").unwrap_err(), + ] { + let message = format!("{error:#}"); + assert!(message.contains("\"n\"") && message.contains("tests.slot_unit") && message.contains("read-only"), "{message}"); + assert!(!message.contains("CANARY"), "{message}"); + } + assert_eq!(cache.snapshot_envelopes(), held); +} + +#[test] +fn a_corrupt_held_payload_is_refused_naming_the_key_never_its_bytes() { + let mut cache = registered(); + cache.put_raw_envelope(envelope("broken", rmp_serde::to_vec(&("CANARY-id", "CANARY-version")).unwrap())).unwrap(); + assert!(!cache.is_writable(UnionSurface::TYPE, "broken").unwrap()); + let error = cache.delete::("broken").unwrap_err(); + let message = format!("{error:#}"); + assert!(message.contains("\"broken\"") && message.contains("backup") && message.contains("store"), "{message}"); + assert!(!message.contains("build that wrote it"), "{message}"); + assert!(!message.contains("CANARY"), "{message}"); + assert!(cache.is_writable("tests.unregistered", "broken").is_err()); +} + +fn held_marked_and_clean() -> (CultCache, tempfile::TempDir, std::path::PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let (cache, path) = open_copy("extra-slot-store.cc", dir.path()); + (cache, dir, path) +} + +#[test] +fn load_soa_does_not_replace_or_drop_a_marked_record() { + use cultcache_rs::CultCacheSoaTable; + let (mut cache, _dir, path) = held_marked_and_clean(); + let all = cache.snapshot_envelopes(); + let held = all.iter().find(|held| held.key == "marked").unwrap().clone(); + let without: Vec<_> = all.iter().filter(|held| held.key != "marked").cloned().collect(); + let mut replaced = all.clone(); + replaced.iter_mut().find(|held| held.key == "marked").unwrap().payload = rmp_serde::to_vec(&constructed()).unwrap(); + for table in [without, replaced] { + let error = cache.load_soa(CultCacheSoaTable::from_envelopes(table)).unwrap_err(); + assert_refused_with(&error, "marked", &mark("Composition", 3, 2, 1)); + assert_eq!(cache.snapshot_envelopes(), all, "a refused load changed the image"); + } + // The same record carried unchanged, with another key added, loads. + let mut kept = all.clone(); + kept.push(envelope("added", rmp_serde::to_vec(&constructed()).unwrap())); + cache.load_soa(CultCacheSoaTable::from_envelopes(kept)).unwrap(); + assert_eq!(cache.snapshot_envelopes().iter().find(|held| held.key == "marked"), Some(&held)); + // A clean record may be dropped by a load. + let no_clean: Vec<_> = cache.snapshot_envelopes().into_iter().filter(|held| held.key != "clean").collect(); + cache.load_soa(CultCacheSoaTable::from_envelopes(no_clean)).unwrap(); + // The marked record is still on disk after the next batch. + let rewrite = envelope("added", rmp_serde::to_vec(&UnionSurface { version: 5, ..constructed() }).unwrap()); + cache.put_prepared_batch(vec![rewrite]).unwrap(); + let mut reopened = registered(); + reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + reopened.pull_all_backing_stores().unwrap(); + assert_eq!(reopened.snapshot_envelopes().iter().find(|held| held.key == "marked"), Some(&held)); +} + +#[test] +fn is_writable_agrees_with_the_writes_for_foreign_keys_and_other_types() { + let dir = tempfile::tempdir().unwrap(); + // Only UnionSurface registered: the store's unit-arm record is foreign here. + let path = dir.path().join("extra-slot-store.cc"); + std::fs::write(&path, without_variant_row(&vector("extra-slot-store.cc"))).unwrap(); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); + assert!(cache.put("unit-arm", &constructed()).is_err()); + assert!(!cache.is_writable(UnionSurface::TYPE, "unit-arm").unwrap()); + assert!(cache.is_writable(UnionSurface::TYPE, "unheld").unwrap()); + + // Another registered type at a key a record of this one holds. + let (mut cache, _dir, _) = held_marked_and_clean(); + let other = SlotUnit { name: "clean".into(), outcome: UnitOutcome::Denied }; + assert!(cache.put("clean", &other).is_err()); + assert!(!cache.is_writable(SlotUnit::TYPE, "clean").unwrap()); + assert!(!cache.is_writable(SlotUnit::TYPE, "marked").unwrap()); + assert!(cache.is_writable(UnionSurface::TYPE, "clean").unwrap()); +} + +#[test] +fn a_marked_key_refuses_a_batch_wherever_it_sits() { + let (mut cache, _dir, path) = held_marked_and_clean(); + let before = std::fs::read(&path).unwrap(); + let marked = envelope("marked", rmp_serde::to_vec(&constructed()).unwrap()); + let clean = |key: &str| envelope(key, rmp_serde::to_vec(&constructed()).unwrap()); + for batch in [ + vec![marked.clone(), clean("a"), clean("b")], + vec![clean("a"), marked.clone(), clean("b")], + vec![clean("a"), clean("b"), marked.clone()], + ] { + let error = cache.put_prepared_batch(batch).unwrap_err(); + assert_refused_with(&error, "marked", &mark("Composition", 3, 2, 1)); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert!(cache.snapshot_envelopes().iter().all(|held| held.key != "a" && held.key != "b")); + } +} + +#[test] +fn a_marked_decode_leaves_no_scope_open() { + assert_eq!(cultcache_rs::__enter_slots(), 0); + decode_payload_marked::(&vector("extra-slot-record.msgpack")).unwrap(); + assert_eq!(cultcache_rs::__enter_slots(), 0, "a marked decode left its scope open on this thread"); + let _ = decode_payload_marked::(&vector("depth-skipped-past-limit.msgpack")); + assert_eq!(cultcache_rs::__enter_slots(), 0); +} + +#[test] +fn is_writable_is_false_for_a_type_with_no_home_store() { + let dir = tempfile::tempdir().unwrap(); + let mut cache = registered(); + // One store, routed to SlotUnit only: UnionSurface has no home. + cache.add_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc")), [SlotUnit::TYPE]).unwrap(); + assert!(cache.put("k", &constructed()).is_err()); + assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap()); + assert!(cache.is_writable(SlotUnit::TYPE, "k").unwrap()); +} + +#[test] +fn load_soa_refuses_a_same_length_replacement_of_a_marked_record() { + use cultcache_rs::CultCacheSoaTable; + let (mut cache, _dir, _) = held_marked_and_clean(); + let all = cache.snapshot_envelopes(); + let marked = all.iter().find(|held| held.key == "marked").unwrap(); + // The clean record's bytes padded to the marked record's length: same length, different bytes. + let mut same_length = vec![0xc0; marked.payload.len()]; + same_length[..2].copy_from_slice(&[0x91, 0xa1]); + assert_eq!(same_length.len(), marked.payload.len()); + let mut replaced = all.clone(); + replaced.iter_mut().find(|held| held.key == "marked").unwrap().payload = same_length; + let error = cache.load_soa(CultCacheSoaTable::from_envelopes(replaced)).unwrap_err(); + assert_refused_with(&error, "marked", &mark("Composition", 3, 2, 1)); + assert_eq!(cache.snapshot_envelopes(), all); +} diff --git a/packages/cultcache-rs/tests/union_parity.rs b/packages/cultcache-rs/tests/union_parity.rs index 24fe999a..c51bd3d8 100644 --- a/packages/cultcache-rs/tests/union_parity.rs +++ b/packages/cultcache-rs/tests/union_parity.rs @@ -1,244 +1,17 @@ //! The Rust runtime held to tests/vectors/union-parity, written by the C# reference. //! Bytes and outcomes are read from the committed files; only the constructed value is spelled here. -use std::path::PathBuf; - use cultcache_rs::CultCache; use cultcache_rs::CacheBackingStore; use cultcache_rs::CultCacheEnvelope; use cultcache_rs::MAX_PAYLOAD_DEPTH; use cultcache_rs::PayloadTooDeep; use cultcache_rs::check_payload_depth; -use cultcache_rs::CultSlots; -use cultcache_rs::CultUnion; use cultcache_rs::DatabaseEntry; use cultcache_rs::SingleFileMessagePackBackingStore; -#[derive(Clone, Debug, PartialEq, CultUnion)] -enum Node { - #[cultcache(key = 0)] - Box(BoxArm), - #[cultcache(key = 1)] - Text(TextArm), - #[cultcache(key = 2)] - Use(UseArm), - #[cultcache(key = 5)] - Image(ImageArm), - #[cultcache(key = 6)] - Wrap(WrapArm), -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct WrapArm { - #[cultcache(key = 0)] - inner: Option>, - #[cultcache(key = 1)] - id: String, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct BoxArm { - #[cultcache(key = 0)] - id: String, - #[cultcache(key = 1)] - children: Vec, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct TextArm { - #[cultcache(key = 0)] - id: String, - #[cultcache(key = 1)] - content: String, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct UseArm { - #[cultcache(key = 0)] - id: String, - #[cultcache(key = 1)] - composition: Composition, - #[cultcache(key = 2)] - slots: Vec>, -} - -// Slot 2 is retired: a gap written nil. -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct ImageArm { - #[cultcache(key = 0)] - id: String, - #[cultcache(key = 1)] - uri: String, - #[cultcache(key = 3)] - alt: String, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct Composition { - #[cultcache(key = 0)] - name: String, - #[cultcache(key = 1)] - body: Option>, -} - -#[derive(Clone, Debug, PartialEq, CultUnion)] -enum Outcome { - #[cultcache(key = 0)] - Pending(PendingArm), - #[cultcache(key = 1)] - Completed(CompletedArm), - #[cultcache(key = 2)] - Denied(IdArm), - #[cultcache(key = -2147483648)] - Floor(IdArm), - #[cultcache(key = 2147483647)] - Ceiling(IdArm), -} - -// The codec's unit arm (no members, written [key, []]), kept apart from the golden: a document list's arms carry ids. -#[derive(Clone, Debug, PartialEq, CultUnion)] -enum UnitOutcome { - #[cultcache(key = 2)] - Denied, - #[cultcache(key = -2147483648)] - Floor, - #[cultcache(key = 2147483647)] - Ceiling, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct PendingArm { - #[cultcache(key = 0)] - message: String, - #[cultcache(key = 1)] - id: String, -} - -// An arm with no members of its own: just its element id. -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct IdArm { - #[cultcache(key = 0)] - id: String, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct CompletedArm { - #[cultcache(key = 0)] - source_version: i32, - #[cultcache(key = 1)] - id: String, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct NamedArm { - #[cultcache(key = 0)] - name: String, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct UsesArm { - #[cultcache(key = 0)] - name: String, - #[cultcache(key = 1)] - uses: i32, -} - -#[derive(Clone, Debug, PartialEq, CultSlots)] -struct CountArm { - #[cultcache(key = 0)] - name: String, - #[cultcache(key = 1)] - count: i32, -} - -#[derive(Clone, Debug, PartialEq, CultUnion)] -enum Item { - #[cultcache(key = 0)] - Gem(NamedArm), - #[cultcache(key = 1)] - Tool(UsesArm), - #[cultcache(key = 2)] - Part(CountArm), -} - -// The same arms as Item under its own keys: key 2 is unknown to it. -#[derive(Clone, Debug, PartialEq, CultUnion)] -enum CraftedItem { - #[cultcache(key = 0)] - Tool(UsesArm), - #[cultcache(key = 1)] - Part(CountArm), -} - -#[derive(Clone, Debug, PartialEq, DatabaseEntry)] -#[cultcache(type = "tests.union_surface", schema = "tests.union_surface.v1")] -struct UnionSurface { - #[cultcache(key = 0)] - id: String, - #[cultcache(key = 1)] - version: i32, - #[cultcache(key = 2)] - root: Option, - #[cultcache(key = 3)] - outcomes: Vec, - #[cultcache(key = 4)] - part: Option, - #[cultcache(key = 5)] - crafted: Option, - #[cultcache(key = 6)] - missing: Option, -} - -fn text(id: &str, content: &str) -> Node { - Node::Text(TextArm { id: id.into(), content: content.into() }) -} - -fn image(id: &str, uri: &str, alt: &str) -> Node { - Node::Image(ImageArm { id: id.into(), uri: uri.into(), alt: alt.into() }) -} - -fn boxed(id: &str, children: Vec) -> Node { - Node::Box(BoxArm { id: id.into(), children }) -} - -fn constructed() -> UnionSurface { - let inner = boxed("0000000000d0", vec![text("0000000000d1", "x"), image("0000000000d2", "uri://a", "alt")]); - UnionSurface { - id: "surface".into(), - version: 3, - root: Some(boxed( - "0000000000c0", - vec![ - text("0000000000c1", "hello"), - Node::Use(UseArm { - id: "0000000000c2".into(), - composition: Composition { name: "comp".into(), body: Some(std::boxed::Box::new(inner)) }, - slots: vec![Some(text("0000000000e1", "slot")), None, Some(boxed("0000000000e2", vec![]))], - }), - image("0000000000c3", "uri://b", ""), - ], - )), - outcomes: vec![ - Outcome::Pending(PendingArm { message: "wait".into(), id: "0000000000a1".into() }), - Outcome::Completed(CompletedArm { source_version: 7, id: "0000000000a2".into() }), - Outcome::Denied(id_arm("0000000000a3")), - ], - part: Some(Item::Tool(UsesArm { name: "hammer".into(), uses: 12 })), - crafted: Some(CraftedItem::Part(CountArm { name: "gear".into(), count: 4 })), - missing: None, - } -} - -fn id_arm(id: &str) -> IdArm { - IdArm { id: id.into() } -} - -fn vector(name: &str) -> Vec { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../tests/vectors/union-parity") - .join(name); - std::fs::read(&path).unwrap_or_else(|error| panic!("{}: {error}", path.display())) -} +mod common; +use common::*; fn decode(bytes: &[u8]) -> Result { rmp_serde::from_slice(bytes).map_err(|error| error.to_string()) @@ -325,11 +98,10 @@ fn an_unknown_arm_lists_the_declared_keys_and_never_echoes_the_payload() { #[test] fn a_union_that_is_not_a_key_and_slots_pair_is_refused_naming_the_enum() { - let cases: [(&str, Vec); 4] = [ + let cases: [(&str, Vec); 3] = [ ("an empty array", vec![0x90]), ("a trailing element", rmp_serde::to_vec(&(1u64, (7u8,), 0u8)).unwrap()), ("a string key", rmp_serde::to_vec(&("CANARY-key", (7u8,))).unwrap()), - ("a unit arm with slots", rmp_serde::to_vec(&(2u64, ("CANARY-slot",))).unwrap()), ]; for (label, bytes) in cases { let message = rmp_serde::from_slice::(&bytes).unwrap_err().to_string(); diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index 33cb4858..18a02fcb 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -732,6 +732,18 @@ impl CultMeshNode { let cultnet_rs::CultNetMessage::SnapshotResponseRaw { documents, .. } = response else { anyhow::bail!("expected cultnet.snapshot_response_raw.v0"); }; + self.apply_raw_snapshot_documents(documents) + } + + /// Applies each snapshot document of a registered schema with `put_raw_envelope`. The first refusal ends the pull + /// and the documents before it stay applied; a key this node holds read-only (unread slots from a newer peer, or a + /// payload this build cannot read) is refused like any other. C# applies a raw snapshot as one cache commit, so a + /// refusal there applies nothing: this is a divergence, owed by follow-up mesh-snapshot-atomic (stage the pull through + /// `put_prepared_batch`). + fn apply_raw_snapshot_documents( + &mut self, + documents: Vec, + ) -> Result { let mut applied = 0usize; for document in documents { let Some(document_type) = @@ -1199,6 +1211,47 @@ mod tests { Note => "cultmesh.test.note.v0", }); + fn raw_note(key: &str, payload: Vec) -> cultnet_rs::CultNetRawDocumentRecord { + cultnet_rs::CultNetRawDocumentRecord { + schema_id: "cultmesh.test.note.v0".to_string(), + record_key: key.to_string(), + stored_at: "2026-01-01T00:00:00Z".to_string(), + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } + } + + #[test] + fn a_snapshot_document_for_a_read_only_key_ends_the_pull_unlike_csharp_which_applies_nothing() -> Result<()> { + let temp = tempfile::tempdir()?; + let mut node = CultMesh::create_node(temp.path().join("m.cc"), TestDocuments, Default::default())?; + // A newer peer's note: one slot past the declared one. + let newer = rmp_serde::to_vec(&("newer body", "extra slot"))?; + let plain = |body: &str| rmp_serde::to_vec(&(body,)); + assert_eq!(node.apply_raw_snapshot_documents(vec![raw_note("held", newer)])?, 1); + assert_eq!(node.get_required::("held")?.body, "newer body"); + + let error = node + .apply_raw_snapshot_documents(vec![ + raw_note("before", plain("b")?), + raw_note("held", plain("overwrite")?), + raw_note("after", plain("a")?), + ]) + .unwrap_err(); + let conflict = error.downcast_ref::().expect("typed refusal"); + assert_eq!(conflict.record_key, "held"); + assert!(conflict.unread_slots.is_some()); + assert!(!error.to_string().contains("extra slot")); + assert_eq!(node.get_required::("before")?.body, "b"); + assert_eq!(node.get_required::("held")?.body, "newer body"); + assert!(node.get::("after")?.is_none()); + Ok(()) + } + #[test] fn node_round_trips_registered_documents() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/packages/cultnet-rs/src/replication.rs b/packages/cultnet-rs/src/replication.rs index 743a28fd..7816e7dd 100644 --- a/packages/cultnet-rs/src/replication.rs +++ b/packages/cultnet-rs/src/replication.rs @@ -146,6 +146,7 @@ impl CultNetDocumentRegistry { schema_id: binding.schema_id.clone(), schema_names: vec![holder.document_type.clone(), binding.document_type.clone()], record_key: String::new(), + unread_slots: None, })); } self.bindings_by_schema_id From 676f611c1fe6c057b59e718e592e201aca2b4906 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 05:03:29 +0000 Subject: [PATCH 3/3] TypeScript: a record with unread slots is held read-only (slot-readonly-ts) (#128) --- packages/cultcache-ts/CHANGELOG.md | 1 + packages/cultcache-ts/README.md | 7 +- packages/cultcache-ts/src/browser.ts | 1 + packages/cultcache-ts/src/codec.ts | 70 ++- packages/cultcache-ts/src/cult-cache.ts | 179 +++++-- .../cultcache-ts/src/document-formatter.ts | 9 +- packages/cultcache-ts/src/document.ts | 17 +- packages/cultcache-ts/src/store-format.ts | 7 +- packages/cultcache-ts/src/types.ts | 20 + .../cultcache-ts/test/slot-readonly.test.ts | 461 ++++++++++++++++++ .../cultcache-ts/test/union-parity.test.ts | 202 +------- .../test/union-surface-fixture.ts | 240 +++++++++ 12 files changed, 976 insertions(+), 238 deletions(-) create mode 100644 packages/cultcache-ts/test/slot-readonly.test.ts create mode 100644 packages/cultcache-ts/test/union-surface-fixture.ts diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index e71821c8..b91a0535 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`. - The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string. - A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. +- A record whose payload carries a slot past its reader's declared slots, at any depth (the record, a nested record, a list element, a union arm, a unit arm), is now held read-only: it loads and reads, and every `put`, `putEnvelope`, `update` or `delete` of its key is refused with `SchemaConflictError` before a store is touched (cultlib-gaps:ruling:extra-trailing-slots, as the C# reference). A load used to skip the extra slots silently and a later write shed them. The refusal names the schema, the key, the type that was read with extra slots and both slot counts, never a slot value, and carries them as `SchemaConflictError.unreadSlots`. An untouched record keeps its bytes through writes of other keys. Missing trailing slots keep their declared defaults and mark nothing. A `put` never marks (the value is encoded by this runtime); a `putEnvelope` of a payload carrying extra slots lands its bytes and holds the record read-only. A mark at depth 0 is consented to only when the record arrived under a schema id other than the reader's own that the reader lists compatible, and that id's catalog entry lists those slots (a writer that adds a slot and keeps the reader's id is never consented to) (`unreadSlots.explained`); every other mark refuses whatever is declared. New: `CultCache.isWritable(definition, key)` (live, advisory), `CultCache.unreadSlots(definition, key)` and the exported `CultCacheUnreadSlots`. Documents with no slot codec (a plain map, a user formatter) never mark. A `delete` is by key: a type that owns the schema id of the record held at the key as another type removes that one record (one store call, the cache dropped once the store has removed it), and a type that does not own it is refused with `SchemaConflictError`, as a `put` is; `isWritable` answers both. Implied semver: a minor bump (0.16.0) while the package is 0.x; a major one at 1.0 and after. ### Added diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 2096d258..6fea306a 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -371,8 +371,11 @@ const sceneDocument = defineDocumentType({ }); ``` -- A slot is `[name, codec, default?]`. A gap is written nil and skipped on read; an extra trailing slot is skipped; a - slot missing from the payload (or nil, when a default is declared) takes its default, and is refused without one. +- A slot is `[name, codec, default?]`. A gap is written nil and skipped on read; an extra trailing slot is read past and + marks the record read-only (every write or removal of its key is refused with `SchemaConflictError`, `cache.isWritable` + asks first). The one writable case is a depth-0 extra slot of a record that arrived under a schema id other than the reader's own, which the + reader lists in `compatibleSchemaIds`, whose catalog entry lists those slots: declaring that id consents to the loss, and a write restamps + the record. A nested slot, or a writer that keeps the reader's own id, is never consented to. A slot missing from the payload (or nil, when a default is declared) takes its default, and is refused without one. - A union refuses an undeclared arm key on read, naming the union, the key and the declared keys. Nothing falls back to a default arm. An arm with no slots is `codecs.unit` and is written `[key, []]`. - A refusal names the union, the field and the slot; it never echoes the value it refused. diff --git a/packages/cultcache-ts/src/browser.ts b/packages/cultcache-ts/src/browser.ts index 668fc0b8..98995acb 100644 --- a/packages/cultcache-ts/src/browser.ts +++ b/packages/cultcache-ts/src/browser.ts @@ -34,6 +34,7 @@ export type { CultCacheIndexScalar, InferCultCacheSchemaValue, CultCacheStoreRegistration, + CultCacheUnreadSlots, PushAllOptions, } from "./types"; export type { diff --git a/packages/cultcache-ts/src/codec.ts b/packages/cultcache-ts/src/codec.ts index 9b885159..a2dd7b15 100644 --- a/packages/cultcache-ts/src/codec.ts +++ b/packages/cultcache-ts/src/codec.ts @@ -198,6 +198,66 @@ type SlotSpec = type SlotsSpec = Readonly }[keyof TValue & string]>>; +// The slots a payload decode read past, noted by `slots` and the unit arm while the one decode entry (decodeNoting) has a scope +// open, and read out by that entry. Module-scoped as the reference's is thread-bound: a decode is synchronous, and a decode +// outside the entry (a codec called directly, a CultMesh read) notes nothing. No codec signature carries it. +export interface NotedSlots { + /** The first type read with extra slots, outermost first. */ + readonly typeName: string; + readonly slotCount: number; + readonly declaredCount: number; + readonly occurrences: number; + /** True when every noted object was the document's own: a union adds no depth, each record or arm inside a field adds one. */ + readonly depth0Only: boolean; +} + +interface SlotScope { + depth: number; + found?: { typeName: string; slotCount: number; declaredCount: number; occurrences: number; depth0Only: boolean }; +} + +let scope: SlotScope | undefined; + +/** The one decode entry: runs the decode with a scope open and returns its value with what it noted. */ +export function decodeNoting(decode: () => T): { value: T; unreadSlots?: NotedSlots } { + const outer = scope; + const mine: SlotScope = { depth: 0 }; + scope = mine; + try { + const value = decode(); + return mine.found ? { value, unreadSlots: mine.found } : { value }; + } finally { + scope = outer; + } +} + +function noteExtraSlots(typeName: string, slotCount: number, declaredCount: number): void { + if (!scope) return; + const found = scope.found; + if (found) { + found.occurrences += 1; + found.depth0Only &&= scope.depth === 0; + } else { + scope.found = { typeName, slotCount, declaredCount, occurrences: 1, depth0Only: scope.depth === 0 }; + } +} + +// What a record inside a field reads at: one deeper than the record holding the field. +function nested(read: () => T): T { + if (!scope) return read(); + scope.depth += 1; + try { + return read(); + } finally { + scope.depth -= 1; + } +} + +// The reference's type name for a mark: the type, not its namespace or the type it is nested in. +function simpleTypeName(typeName: string | undefined): string { + return typeName === undefined ? "record" : typeName.slice(Math.max(typeName.lastIndexOf("."), typeName.lastIndexOf("+")) + 1); +} + const INT32_MIN = -(2 ** 31); const INT32_MAX = 2 ** 31 - 1; @@ -232,7 +292,7 @@ function pascalCase(name: string): string { } // A positional record: slot `key` holds field `name`. Gaps are written nil and skipped on read, extra trailing slots -// are skipped, and a slot missing from the payload takes its declared default or is refused. A default fills only a +// are read past and noted (decodeNoting): the record that carried them is read-only, never rewritten without them, and a slot missing from the payload takes its declared default or is refused. A default fills only a // missing trailing slot, never a nil: as in the reference, a nil in a reference slot (string, bytes, record, union, // list) reads as null and writes back as nil, so a record the reference wrote re-encodes byte-identical, and a nil in // a value slot (int, float, bool) is refused unless the slot is nullable. @@ -281,6 +341,9 @@ export function slots(spec: SlotsSpec, typeName?: decode(raw) { expect(raw, "an array of slots", Array.isArray(raw)); const input = raw as unknown[]; + if (input.length > length) { + noteExtraSlots(simpleTypeName(typeName), input.length, length); + } const out: Record = {}; for (const field of fields) { const present = field.key < input.length; @@ -291,7 +354,7 @@ export function slots(spec: SlotsSpec, typeName?: ? defaultOf(field) : input[field.key] === null && !field.codec.valueType ? null - : at(`${field.name} (slot ${field.key})`, () => field.codec.decode(input[field.key])); + : at(`${field.name} (slot ${field.key})`, () => nested(() => field.codec.decode(input[field.key]))); } return out as TValue; }, @@ -363,6 +426,9 @@ export function union( if (arm === undefined) { throw new Error(`Union ${name}: unknown arm key ${key} (declared keys: ${declared}).`); } + if (arm.body === unit && raw[1].length > 0) { + noteExtraSlots(arm.armName, raw[1].length, 0); + } const fields = arm.body === unit ? {} : at(`arm ${arm.armName}`, () => (arm.body as SlotsCodec).decode(raw[1])); return { kind: arm.armName, ...fields } as unknown as TValue; }, diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 443c0c18..6cf7a8cb 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,6 +1,6 @@ import { declaresRole, - decodeDocumentPayload, + decodeDocumentPayloadMarked, documentPayloadBytes, documentPayloadFromBytes, encodeDocumentPayload, @@ -8,6 +8,7 @@ import { schemaIdentityOf, } from "./document"; import { resolveDocumentFormatter } from "./document-formatter"; +import type { NotedSlots } from "./codec"; import type { DocumentPayload } from "./document"; import { SchemaConflictError, schemaIdForEnvelope, writtenOwns } from "./store-format"; import type { @@ -20,6 +21,7 @@ import type { CultCacheEnvelope, CultCacheSchemaCatalogEntry, CultCacheStoreRegistration, + CultCacheUnreadSlots, } from "./types"; type Accessor = (value: unknown) => string | undefined; @@ -27,7 +29,7 @@ type Accessor = (value: unknown) => string | undefined; // A registered definition reads and writes its documents through the owner, on payloads the cache has cloned. type RegisteredFormatter = { encode(value: unknown): DocumentPayload; - decode(payload: DocumentPayload): unknown; + decode(payload: DocumentPayload): { value: unknown; unreadSlots?: NotedSlots }; }; type RegisteredDefinition = { @@ -50,6 +52,10 @@ type HydratedEntry = CultCacheEnvelope & { catalogEntry: NonNullable; value: unknown; lookups: EntryLookups; + // What the payload decode read past: set when the record was admitted, and it makes the record read-only unless explained. + unreadSlots?: CultCacheUnreadSlots; + // The schema id the record arrived under, before it was admitted under its reader's: what an unread-slots refusal names. + storedSchemaId?: string; }; // Everything a pull replaces. Built whole, then swapped in with one assignment. @@ -84,6 +90,20 @@ type DerivedTypeLookups = { readonly indexMaps: Map>; }; +// The consent a compatible-id declaration gives (cultlib-gaps:ruling:slot-mark-consent): a mark is explained only when the record +// arrived under an id other than the reader's own that the reader lists compatible, it is at +// depth 0 and the schema comparison of the catalog entry the record arrived with against the reader's own reported the slots it +// carries as ignored, so its slot count is at most the highest ignored slot plus one. A record written or put through the cache +// arrives under the reader's own entry, which ignores nothing, so no mark of it is explained. +function explainedBy(noted: NotedSlots, reader: CultCacheSchemaCatalogEntry, arrivedId: string, arrived: CultCacheSchemaCatalogEntry | undefined): boolean { + if (arrivedId === reader.schemaId || !(reader.compatibleSchemaIds ?? []).includes(arrivedId)) { + return false; + } + const declared = new Set((reader.members ?? []).map((member) => member.slot)); + const ignored = (arrived?.members ?? []).map((member) => member.slot).filter((slot) => !declared.has(slot)); + return noted.depth0Only && noted.slotCount <= Math.max(-1, ...ignored) + 1; +} + function emptyHydratedState(): HydratedState { return { entries: new Map(), @@ -312,7 +332,7 @@ export class CultCache { } const payload = this.#cloneBytes(entry.payload); - const value = registered.formatter.decode(payload); + const { value, unreadSlots } = registered.formatter.decode(payload); let admitted: CultCacheEnvelope = { ...entry, type, payload }; if (registered.global) { const persistedKey = persistedGlobalKeys.get(type); @@ -327,7 +347,7 @@ export class CultCache { admitted = { ...admitted, key: CultCache.GLOBAL_KEY }; } } - const hydrated = this.#admitEntry(next, registered, admitted, value); + const hydrated = this.#admitEntry(next, registered, admitted, value, unreadSlots); const other = this.#heldUnderOtherType(next, type, hydrated.key); if (other !== undefined) { throw new Error(`Record ${hydrated.key} is held by two stores, as type "${other.type}" and as type "${type}".`); @@ -518,10 +538,9 @@ export class CultCache { if (!envelope.storedAt || envelope.storedAt.trim().length === 0) { throw new Error(`CultCache envelope storedAt for type "${definition.type}" must be non-empty.`); } - this.#requireGlobalKey(registered, envelope.key); const payload = this.#cloneBytes(envelope.payload); - const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue; + const { value: parsed, unreadSlots } = registered.formatter.decode(payload) as { value: CultCacheDocumentValue; unreadSlots?: NotedSlots }; const entry: CultCacheEnvelope = { key: envelope.key, type: envelope.type, @@ -532,7 +551,7 @@ export class CultCache { schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, }; - await this.#writeNow(registered, entry, parsed); + await this.#writeNow(registered, entry, parsed, unreadSlots); return parsed; }); } @@ -585,6 +604,29 @@ export class CultCache { }); } + /** What the record's payload carried past this build's declared slots, or undefined: the held record is read-only unless `explained`. */ + unreadSlots(definition: AnyCultCacheDocumentDefinition, key: string): CultCacheUnreadSlots | undefined { + return this.#hydrated.entries.get(this.#entryId(this.#requireDefinition(definition).definition.type, key))?.unreadSlots; + } + + /** + * Whether a write of the record's own type at the key would be let through: it asks the judge #writeNow asks, of the record the + * cache holds (a key a store carries foreign, a record of another type the write does not own or cannot replace, a global at + * another key, a record carrying unread slots nothing consents to). It answers the WRITE form. A removal asks the judge's + * removal form and differs in two states: an owner's `delete` of a record held in another store than its own home succeeds + * where this is false (a write would push to the wrong store), and `delete` of a global type at a key other than __global__ is + * the plain false of an unheld key where this is false. Live and advisory: the write or removal still decides. + */ + isWritable(definition: AnyCultCacheDocumentDefinition, key: string): boolean { + const registered = this.#requireDefinition(definition); + try { + this.#judge(registered, key, this.#probe(registered, key)); + return true; + } catch { + return false; + } + } + // What the last pull found that no registered type claims, by key ordinal. A later registration claims a record at the next pull. foreignRecords(): readonly CultCacheForeignRecord[] { return [...this.#hydrated.foreign].sort((left, right) => compareOrdinal(left.key, right.key)); @@ -680,9 +722,8 @@ export class CultCache { value: CultCacheDocumentValue, ): Promise> { const registered = this.#requireDefinition(definition); - this.#requireGlobalKey(registered, key); const payload = registered.formatter.encode(value); - const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue; + const { value: parsed, unreadSlots } = registered.formatter.decode(payload) as { value: CultCacheDocumentValue; unreadSlots?: NotedSlots }; const entry: CultCacheEnvelope = { key, type: definition.type, @@ -691,26 +732,16 @@ export class CultCache { schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, }; - await this.#writeNow(registered, entry, parsed); + await this.#writeNow(registered, entry, parsed, unreadSlots); return parsed; } // Validates the whole write (home, global singleton, name and index accessors) before the store // is touched; once the store accepts, applying to the cache has no fallible step. - async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise { + async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown, unreadSlots?: NotedSlots): Promise { const home = this.#homeStore(entry.type); - const retyped = this.#refuseRetype(entry); - if (retyped !== undefined && this.#homeStore(retyped.type) !== home) { - // The write owns the held record's id and would replace it, but the held record is in another store; the push - // reaches only this type's home, so the old record would stay in its own file and load as a second holder. - throw new SchemaConflictError( - schemaIdForEnvelope(entry), - [retyped.type, entry.type], - entry.key, - `CultCache holds record "${entry.key}" as type "${retyped.type}" in another store than the home of type "${entry.type}", so the rename cannot replace it; remove the record first.`, - ); - } - const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value); + const retyped = this.#judge(registered, entry.key, entry); + const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value, unreadSlots); this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups); const legacy = this.#hydrated.legacyGlobals.get(entry.type); await home?.push(entry); @@ -725,6 +756,85 @@ export class CultCache { this.#installEntry(this.#hydrated, registered, hydrated); } + // An envelope of the definition's type at the key, carrying nothing: what the judge asks about a write or a removal by that type. + #probe(registered: RegisteredDefinition, key: string): CultCacheEnvelope { + return { + key, type: registered.definition.type, payload: documentPayloadFromBytes(new Uint8Array(0)), storedAt: "", + schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, + }; + } + + // The one judge of whether a key may be written (a written envelope) or removed (none), before any store is touched: it throws the + // refusal, and #writeNow, #deleteNow and isWritable all ask it, so the query cannot disagree with the write. For a write it + // returns the held record of another type that the write owns and so restamps, if any. A refusal names keys and schemas, + // never a payload or a slot value. + #judge(registered: RegisteredDefinition, key: string, written?: CultCacheEnvelope): HydratedEntry | undefined { + const type = registered.definition.type; + const held = this.#hydrated.entries.get(this.#entryId(type, key)); + if (written === undefined) { + // A removal is by key: the record it removes is the one held at the key, of this type or of another type whose schema id + // this type owns (the record a write of this type would replace). A held record of a type it does not own is refused. + this.#refuseForeignKey(key, type, "removed"); + this.#refuseUnread(held); + if (held !== undefined) { + return undefined; + } + const other = this.#heldUnderOtherType(this.#hydrated, type, key); + if (other === undefined) { + return undefined; + } + if (!writtenOwns(this.#probe(registered, key), schemaIdForEnvelope(other))) { + throw new SchemaConflictError( + registered.catalogEntry.schemaId, + [other.type, type], + key, + `CultCache holds record "${key}" as type "${other.type}", so type "${type}" cannot remove it.`, + ); + } + this.#refuseUnread(other); + return other; + } + this.#requireGlobalKey(registered, key); + const retyped = this.#refuseRetype(written); + this.#refuseUnread(held); + this.#refuseUnread(retyped); + if (retyped !== undefined && this.#homeStore(retyped.type) !== this.#homeStore(type)) { + // The write owns the held record's id and would replace it, but the held record is in another store; the push + // reaches only this type's home, so the old record would stay in its own file and load as a second holder. + throw new SchemaConflictError( + schemaIdForEnvelope(written), + [retyped.type, type], + key, + `CultCache holds record "${key}" as type "${retyped.type}" in another store than the home of type "${type}", so the rename cannot replace it; remove the record first.`, + ); + } + return retyped; + } + + // The one refusal of a record held read-only for the slots its payload carries past this build's: a write or a removal of + // its key. It names the type and the counts, never a slot value. A mark that is explained (see #admitEntry) is consented to. + #unreadRefusal(held: HydratedEntry | undefined): SchemaConflictError | undefined { + const unread = held?.unreadSlots; + if (held === undefined || unread === undefined || unread.explained) { + return undefined; + } + return new SchemaConflictError( + held.storedSchemaId ?? held.schemaId ?? held.type, + [held.catalogEntry.schemaName], + held.key, + `${held.catalogEntry.schemaName} record ${held.key} is read-only: ${unread.typeName} was read with ${unread.slotCount} slots where this build declares ${unread.declaredCount} ` + + `(${unread.occurrences} object(s) in the record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or write it from that build.`, + unread, + ); + } + + #refuseUnread(held: HydratedEntry | undefined): void { + const refusal = this.#unreadRefusal(held); + if (refusal !== undefined) { + throw refusal; + } + } + // The one key-identity judge of the cache: may this write land at its key, given what the cache holds there under // every registered type and the keys its stores carry foreign. The store judges the file with the same predicate // (writtenOwns); this judges the cache, which a store-less cache has no other judge for. Returns the held record of @@ -782,18 +892,20 @@ export class CultCache { key: string, ): Promise { const registered = this.#requireDefinition(definition); - const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key)); + // variants:ruling:unheld-removal-refused-at-commit: a key a store carries foreign is refused, a key held by nobody is the + // existing false; a key held as another type is removed by an owner of its id and refused to any other type. + // Removal is by key, in one store call: the record held at the key of this type, else the one of another type this type owns. + const owned = this.#judge(registered, key); + const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key)) ?? owned; if (!entry) { - // variants:ruling:unheld-removal-refused-at-commit: a key a store carries foreign is refused, a key held under - // another type or by nobody is the existing false. - this.#refuseForeignKey(key, definition.type, "removed"); return false; } - const home = this.#homeStore(definition.type); - await home?.delete(this.#hydrated.legacyGlobals.get(definition.type) ?? this.#toEnvelope(entry)); - this.#hydrated.legacyGlobals.delete(definition.type); - this.#removeEntry(this.#hydrated, registered, entry); + // The cache drops the record only once the store has removed it. + const home = this.#homeStore(entry.type); + await home?.delete(this.#hydrated.legacyGlobals.get(entry.type) ?? this.#toEnvelope(entry)); + this.#hydrated.legacyGlobals.delete(entry.type); + this.#removeEntry(this.#hydrated, this.#definitions.get(entry.type)!, entry); return true; } @@ -820,7 +932,7 @@ export class CultCache { return { encode: (value: unknown) => this.#cloneBytes(encodeDocumentPayload(definition, value)), - decode: (payload: DocumentPayload) => decodeDocumentPayload(definition, this.#cloneBytes(payload)), + decode: (payload: DocumentPayload) => decodeDocumentPayloadMarked(definition, this.#cloneBytes(payload)), }; } @@ -1000,6 +1112,7 @@ export class CultCache { registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown, + noted?: NotedSlots, ): HydratedEntry { if (registered.global) { const currentGlobalKey = state.globalKeys.get(entry.type); @@ -1020,6 +1133,8 @@ export class CultCache { catalogEntry: registered.catalogEntry, value, lookups, + storedSchemaId: entry.schemaId ?? entry.type, + ...(noted ? { unreadSlots: { ...noted, explained: explainedBy(noted, registered.catalogEntry, entry.schemaId ?? entry.type, entry.catalogEntry) } } : {}), }; } diff --git a/packages/cultcache-ts/src/document-formatter.ts b/packages/cultcache-ts/src/document-formatter.ts index 799b2558..92d45df1 100644 --- a/packages/cultcache-ts/src/document-formatter.ts +++ b/packages/cultcache-ts/src/document-formatter.ts @@ -1,7 +1,7 @@ import { decode, encode } from "@msgpack/msgpack"; import { MAX_PAYLOAD_DEPTH } from "./payload-depth"; -import type { SlotsCodec } from "./codec"; +import { decodeNoting, type NotedSlots, type SlotsCodec } from "./codec"; import type { AnyCultCacheDocumentDefinition, CultCacheDocumentFormatter } from "./types"; import { readWire, writeWire } from "./wire"; @@ -16,6 +16,13 @@ export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinit return definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : plainDocumentFormatter); } +// The one decode of a document's bytes, with the slots it read past. A plain formatter or a user formatter has no slot codec +// and carries everything, so it notes nothing. +export function decodeMarked(definition: AnyCultCacheDocumentDefinition, bytes: Uint8Array): { value: unknown; unreadSlots?: NotedSlots } { + const formatter = resolveDocumentFormatter(definition); + return decodeNoting(() => formatter.decode(bytes)); +} + const plainDocumentFormatter: CultCacheDocumentFormatter = { // The encoder's own depth refusal (100 by default) is set above the limit, so the payload walk decides. encode: (value: unknown) => encode(value, { maxDepth: MAX_PAYLOAD_DEPTH * 2 }), diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 10af96a6..8d295bd4 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -1,4 +1,5 @@ -import { resolveDocumentFormatter } from "./document-formatter"; +import type { NotedSlots } from "./codec"; +import { decodeMarked, resolveDocumentFormatter } from "./document-formatter"; import { checkPayloadDepth } from "./payload-depth"; import type { AnyCultCacheDocumentDefinition, @@ -118,10 +119,20 @@ export function decodeDocumentPayload { + return decodeDocumentPayloadMarked(definition, payload).value; +} + +// The same read with the slots it read past (cult-cache-ts slot-readonly). The depth check runs first: a payload past the +// limit is refused, never marked. +export function decodeDocumentPayloadMarked( + definition: TDefinition, + payload: DocumentPayload, +): { value: CultCacheDocumentValue; unreadSlots?: NotedSlots } { const bytes = documentPayloadBytes(payload); if (writesLibraryMessagePack(definition)) checkPayloadDepth(bytes, `document type "${definition.type}" payload`); - const decoded = resolveDocumentFormatter(definition).decode(bytes); - return definition.schema.parse(decoded) as CultCacheDocumentValue; + const { value: decoded, unreadSlots } = decodeMarked(definition, bytes); + const value = definition.schema.parse(decoded) as CultCacheDocumentValue; + return unreadSlots ? { value, unreadSlots } : { value }; } // The one writer: the definition's schema first, then its formatter, codec or plain map. diff --git a/packages/cultcache-ts/src/store-format.ts b/packages/cultcache-ts/src/store-format.ts index acd10207..885b1c8e 100644 --- a/packages/cultcache-ts/src/store-format.ts +++ b/packages/cultcache-ts/src/store-format.ts @@ -1,6 +1,6 @@ import { decode } from "@msgpack/msgpack"; -import type { CultCacheEnvelope } from "./types"; +import type { CultCacheEnvelope, CultCacheUnreadSlots } from "./types"; export const STORE_FORMAT_VERSION = "cultcache.store.v1"; @@ -83,13 +83,16 @@ export class SchemaConflictError extends Error { readonly schemaId: string; readonly schemaNames: string[]; readonly recordKey: string; + /** Set when the refusal is for unread slots; undefined for every other refusal. */ + readonly unreadSlots?: CultCacheUnreadSlots; - constructor(schemaId: string, schemaNames: string[], recordKey: string, message?: string) { + constructor(schemaId: string, schemaNames: string[], recordKey: string, message?: string, unreadSlots?: CultCacheUnreadSlots) { super(message ?? `Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`); this.name = "SchemaConflictError"; this.schemaId = schemaId; this.schemaNames = schemaNames; this.recordKey = recordKey; + this.unreadSlots = unreadSlots; } } diff --git a/packages/cultcache-ts/src/types.ts b/packages/cultcache-ts/src/types.ts index 6f6c0923..10371053 100644 --- a/packages/cultcache-ts/src/types.ts +++ b/packages/cultcache-ts/src/types.ts @@ -99,6 +99,26 @@ export interface CultCacheEnvelope { catalogEntry?: CultCacheSchemaCatalogEntry; } +/** + * What a payload decode found past the slots this build declares, at any depth: the first type read with extra slots, how many + * slots it carried against how many it declares, and how many objects in the record did. It holds no slot value, so a refusal + * can name it. A record carrying it is read-only unless `explained`. + */ +export interface CultCacheUnreadSlots { + /** The first type read with extra slots (the unit arm's arm name for a unit arm). */ + readonly typeName: string; + readonly slotCount: number; + readonly declaredCount: number; + readonly occurrences: number; + /** True when every mark sat in the document's own object. */ + readonly depth0Only: boolean; + /** + * True only when every mark is at depth 0 and the schema comparison of the id the record was stored under reported every slot + * the mark carries (slotCount is at most the highest reported slot plus one): the loss a compatible-id declaration consents to. + */ + readonly explained: boolean; +} + export interface PushAllOptions { soft?: boolean; } diff --git a/packages/cultcache-ts/test/slot-readonly.test.ts b/packages/cultcache-ts/test/slot-readonly.test.ts new file mode 100644 index 00000000..e3285f07 --- /dev/null +++ b/packages/cultcache-ts/test/slot-readonly.test.ts @@ -0,0 +1,461 @@ +import assert from "node:assert/strict"; +import { copyFile, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { decode, encode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { codecs } from "../src"; +import { CultCache } from "../src/cult-cache"; +import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document"; +import { SingleFileMessagePackBackingStore, readSingleFileStore } from "../src/single-file-messagepack-backing-store"; +import { SchemaConflictError, StoreUnreadableError } from "../src/store-format"; +import type { CacheBackingStore, CultCacheEnvelope, CultCacheUnreadSlots } from "../src/types"; +import { + type Surface, envelopeOf, golden, hex, memoryStore, read, slotUnitDocument, surfaceCodec, surfaceDeclaringDocument, surfaceDocument, surfaceSchema, vectors, +} from "./union-surface-fixture"; + +// cultlib-gaps:cut:slot-readonly-ts. A payload carrying a slot past its reader's declared slots, at any depth, is read, held +// read-only, and every write or removal of its key is refused before a store is touched. The vectors are written by the C# +// reference: tests/vectors/union-parity/README.md, "Extra trailing slot" and "Slot stores". + +const { slots, string } = codecs; +const SLOT_VALUE = "extra-slot"; + +function mark(typeName: string, slotCount: number, declaredCount: number, depth0Only: boolean, explained = false, occurrences = 1): CultCacheUnreadSlots { + return { typeName, slotCount, declaredCount, occurrences, depth0Only, explained }; +} + +// File, then the mark the README table gives for it. +const PAYLOAD_ROWS: [string, CultCacheUnreadSlots][] = [ + ["extra-slot-record.msgpack", mark("UnionSurface", 8, 7, true)], + ["extra-slot-nested-record.msgpack", mark("Composition", 3, 2, false)], + ["extra-slot-arm.msgpack", mark("Box", 3, 2, false)], + ["extra-slot-list-element.msgpack", mark("Text", 3, 2, false)], + ["depth-skipped-at-limit.msgpack", mark("UnionSurface", 8, 7, true)], +]; + +async function readOnlyFiles(name: string): Promise { + const rows = decode(await read(name)) as unknown[][]; + return rows.filter((row) => row[1] === "read-only").map((row) => row[0] as string); +} + +test("the read-only payload vectors are exactly the ones this walker names: four in outcomes, one in bounds", async () => { + const listed = [...(await readOnlyFiles("outcomes.msgpack")), ...(await readOnlyFiles("bounds-outcomes.msgpack"))]; + assert.deepEqual(listed.sort(), PAYLOAD_ROWS.map(([file]) => file).sort()); +}); + +for (const [file, expected] of PAYLOAD_ROWS) { + test(`${file}: read past the extra slot, held read-only with the mark, every write and removal of the key refused`, async () => { + const document = surfaceDocument(); + const entries = [envelopeOf(await read(file))]; + const before = hex(documentPayloadBytes(entries[0]!.payload)); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build(); + await cache.pullAllBackingStores(); + + assert.deepEqual(cache.get(document, "surface"), golden, "the extra slot is read past"); + assert.deepEqual(cache.unreadSlots(document, "surface"), expected); + assert.equal(cache.isWritable(document, "surface"), false); + assert.equal(cache.isWritable(document, "another"), true); + + const refused = (error: unknown) => { + assert.ok(error instanceof SchemaConflictError, "a SchemaConflictError"); + assert.deepEqual(error.unreadSlots, expected); + assert.equal(error.recordKey, "surface"); + assert.ok(error.message.includes(expected.typeName), error.message); + assert.ok(error.message.includes(`read with ${expected.slotCount} slots where this build declares ${expected.declaredCount} (${expected.occurrences} object(s)`), error.message); + assert.ok(!error.message.includes(SLOT_VALUE), "the refusal never holds a slot value"); + return true; + }; + await assert.rejects(() => cache.put(document, "surface", golden), refused); + await assert.rejects(() => cache.update(document, "surface", (value) => ({ ...value!, version: 4 })), refused); + const replacement = envelopeOf(await read("union-surface.msgpack")); + await assert.rejects(() => cache.putEnvelope(document, replacement), refused); + await assert.rejects(() => cache.delete(document, "surface"), refused); + assert.equal(entries.length, 1, "nothing reached the store"); + assert.equal(hex(documentPayloadBytes(entries[0]!.payload)), before, "the held bytes are untouched"); + assert.deepEqual(cache.get(document, "surface"), golden, "the record is still held"); + + await cache.put(document, "another", golden); + assert.equal(entries.length, 2, "another key of the same store writes"); + }); +} + +test("a clean payload and a short one are writable: missing trailing slots keep the defaults and mark nothing", async () => { + const document = surfaceDocument(); + const short = encode([...(decode(await read("union-surface.msgpack")) as unknown[]).slice(0, 6)]); + const entries = [envelopeOf(await read("union-surface.msgpack")), { ...envelopeOf(short), key: "short" }]; + const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build(); + await cache.pullAllBackingStores(); + for (const key of ["surface", "short"]) { + assert.equal(cache.unreadSlots(document, key), undefined); + assert.equal(cache.isWritable(document, key), true); + } + await cache.put(document, "short", golden); +}); + +test("a payload past the nesting limit is refused, never marked", async () => { + const entries = [envelopeOf(await read("depth-skipped-past-limit.msgpack"))]; + const cache = CultCache.builder().withDocumentType(surfaceDocument()).withGenericStore(memoryStore(entries)).build(); + await assert.rejects(() => cache.pullAllBackingStores(), /nesting|depth/iu); +}); + +test("putEnvelope of a payload carrying extra slots lands the bytes and holds the record read-only", async () => { + const document = surfaceDocument(); + const entries: ReturnType[] = []; + const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build(); + const marked = envelopeOf(await read("extra-slot-nested-record.msgpack")); + await cache.putEnvelope(document, marked); + assert.equal(hex(documentPayloadBytes(entries[0]!.payload)), hex(await read("extra-slot-nested-record.msgpack")), "the bytes are carried"); + assert.deepEqual(cache.unreadSlots(document, "surface"), mark("Composition", 3, 2, false)); + assert.equal(cache.isWritable(document, "surface"), false); + await assert.rejects(() => cache.put(document, "surface", golden), SchemaConflictError); +}); + +test("a key a store carries foreign is not writable", async () => { + const document = surfaceDocument(); + const foreign = { ...envelopeOf(await read("union-surface.msgpack")), key: "stranger", type: "tests.unknown", schemaId: "tests.unknown.v1" }; + const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([foreign])).build(); + await cache.pullAllBackingStores(); + assert.equal(cache.isWritable(document, "stranger"), false); + assert.equal(cache.isWritable(document, "other"), true); +}); + +test("a write by a type that owns the held record's id restamps it, and is refused when that record carries unread slots", async () => { + const typed = (type: string, schemaId: string, compatibleSchemaIds: string[] = []) => + defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, compatibleSchemaIds, schema: surfaceSchema, codec: surfaceCodec, name: "id" }); + const older = typed("tests.surface_older", "tests.surface_older.v1"); + const newer = typed("tests.surface_newer", "tests.surface_newer.v1", ["tests.surface_older.v1"]); + for (const [file, lands] of [["union-surface.msgpack", true], ["extra-slot-record.msgpack", false]] as const) { + const held = { ...envelopeOf(await read(file)), type: older.type, schemaId: "tests.surface_older.v1" }; + const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(memoryStore([held])).build(); + await cache.pullAllBackingStores(); + assert.equal(cache.isWritable(newer, "surface"), lands, `${file}: isWritable agrees with the write by the type that owns the id`); + const write = cache.put(newer, "surface", golden); + if (lands) { + await write; + } else { + await assert.rejects(() => write, (error: unknown) => error instanceof SchemaConflictError && error.unreadSlots?.typeName === "UnionSurface"); + } + } +}); + +// The store vectors. TS has no variant records: extra-slot-store.cc carries one (marked-variant), so the whole file is refused +// by name, and the walk runs over the file without that record. +async function copyOf(name: string): Promise { + const path = join(await mkdtemp(join(tmpdir(), "slot-readonly-")), name); + await copyFile(join(vectors, name), path); + return path; +} + +async function withoutVariant(path: string): Promise { + const [format, catalog, records] = decode(await readFile(path)) as [unknown, unknown[], unknown[][]]; + await writeFile(path, encode([format, catalog, records.filter((record) => record[0] !== "marked-variant")])); +} + +type SlotRow = [string, string, boolean, boolean | null, string | null, number | null, number | null, number | null, string | null]; + +async function slotRows(): Promise { + return decode(await read("slot-outcomes.msgpack")) as SlotRow[]; +} + +// The catalog entry the file publishes for a schema: its id and the reference's content hash, which a store rewrites with the +// definition's own. +async function catalogOf(file: string, schemaName: string): Promise<[string, string]> { + const [, catalog] = decode(await read(file)) as [unknown, string[][]]; + const entry = catalog.find((candidate) => candidate[1] === schemaName)!; + return [entry[0]!, entry[3]!]; +} + +async function openStore(file: string, consent: boolean) { + const path = await copyOf(file); + if (file === "extra-slot-store.cc") await withoutVariant(path); + const surface = consent ? surfaceDeclaringDocument() : surfaceDocument(...(await catalogOf(file, "tests.union_surface"))); + const slotUnit = consent ? undefined : slotUnitDocument(...(await catalogOf(file, "tests.slot_unit"))); + const builder = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path)); + const cache = (slotUnit ? builder.withDocumentType(slotUnit) : builder).build(); + await cache.pullAllBackingStores(); + return { path, cache, surface, slotUnit }; +} + +test("extra-slot-store.cc carries a variant, which this runtime refuses by name; the rest of the file is walked", async () => { + const path = await copyOf("extra-slot-store.cc"); + assert.throws(() => readSingleFileStore(path), StoreUnreadableError); +}); + +test("every slot-outcomes row holds: the mark, IsWritable, and the removal refused exactly where the record is not writable", async () => { + const rows = await slotRows(); + assert.equal(rows.length, 8); + let walked = 0; + for (const [file, key, writable, explained, typeName, slots, declared, occurrences, flattenRefusedBy] of rows) { + if (flattenRefusedBy !== null) continue; // a variant: this runtime holds none + walked += 1; + const { path, cache, surface, slotUnit } = await openStore(file!, file === "extra-slot-consent.cc"); + const document = key === "unit-arm" ? slotUnit! : surface; + const label = `${file} ${key}`; + const found = cache.unreadSlots(document, key!); + if (typeName === null) { + assert.equal(found, undefined, label); + } else { + assert.ok(found, label); + assert.deepEqual([found.explained, found.typeName, found.slotCount, found.declaredCount, found.occurrences], [explained, typeName, slots, declared, occurrences], label); + } + assert.equal(cache.isWritable(document, key!), writable, label); + + const original = await readFile(path); + if (writable) { + assert.equal(await cache.delete(document, key!), true, label); + } else { + await assert.rejects( + () => cache.delete(document, key!), + (error: unknown) => error instanceof SchemaConflictError && error.unreadSlots?.typeName === typeName && !error.message.includes(SLOT_VALUE), + label, + ); + assert.equal(Buffer.compare(original, await readFile(path)), 0, `${label}: the file is unchanged`); + } + } + assert.equal(walked, 7); +}); + +test("consent: a write of the explained record lands; the over-consent write is refused; the nested mark is refused", async () => { + const consented = await openStore("extra-slot-consent.cc", true); + const value = consented.cache.get(consented.surface, "consented") as Surface; + assert.deepEqual(value, golden); + await consented.cache.put(consented.surface, "consented", value); + assert.equal(consented.cache.unreadSlots(consented.surface, "consented"), undefined, "the write dropped the explained slot and the mark with it"); + assert.equal(readSingleFileStore(consented.path).envelopes.find((e) => e.key === "consented")!.schemaId, "tests.union_surface.v2"); + + for (const key of ["over-consent", "nested"]) { + const refused = await openStore("extra-slot-consent.cc", true); + await assert.rejects( + () => refused.cache.put(refused.surface, key, golden), + (error: unknown) => error instanceof SchemaConflictError && error.schemaId === "tests.union_surface.wide", + `${key}: the refusal names the id the record was stored under`, + ); + } +}); + +test("carried: after writing another record, the marked record's bytes, schema id, stored-at and every catalog entry are the originals", async () => { + const { path, cache, surface } = await openStore("extra-slot-store.cc", false); + const originals = readSingleFileStore(path); + await cache.put(surface, "clean", golden); + const after = readSingleFileStore(path); + for (const key of ["marked", "unit-arm"]) { + const was = originals.envelopes.find((e) => e.key === key)!; + const now = after.envelopes.find((e) => e.key === key)!; + assert.equal(hex(documentPayloadBytes(now.payload)), hex(documentPayloadBytes(was.payload)), key); + assert.equal(now.schemaId, was.schemaId, key); + assert.equal(now.storedAt, was.storedAt, key); + assert.deepEqual(now.catalogEntry, was.catalogEntry, key); + } +}); + +// A writer that adds a slot and keeps its schema id (TS ids are hand-written; C# ids hash the members): the reader of that +// same id has declared nothing, so it consents to nothing. +test("a wider writer under the reader's own schema id is read-only, and a real round trip does not shed the slot", async () => { + const doc = (columns: 1 | 2, compatibleSchemaIds: string[] = []) => defineDocumentType({ + type: "tests.own_id", + schemaId: "tests.own_id.v1", + schemaName: "tests.own_id", + compatibleSchemaIds, + schema: z.object({ n: z.string(), extra: z.string().optional() }), + codec: columns === 2 + ? slots<{ n: string; extra: string }>({ 0: ["n", string], 1: ["extra", string] }, "Own") + : slots<{ n: string }>({ 0: ["n", string] }, "Own"), + name: "n", + }); + const path = join(await mkdtemp(join(tmpdir(), "slot-readonly-")), "own.cc"); + const wide = doc(2); + const writer = CultCache.builder().withDocumentType(wide).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await writer.put(wide, "k", { n: "n1", extra: "kept" }); + const bytes = await readFile(path); + + for (const compatible of [[], ["tests.own_id.v1"]]) { + const reader = doc(1, compatible); + const cache = CultCache.builder().withDocumentType(reader).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await cache.pullAllBackingStores(); + assert.deepEqual(cache.unreadSlots(reader, "k"), mark("Own", 2, 1, true), "unexplained: the arrived id is the reader's own"); + assert.equal(cache.isWritable(reader, "k"), false); + await assert.rejects(() => cache.put(reader, "k", { n: "n2" }), SchemaConflictError); + await assert.rejects(() => cache.delete(reader, "k"), SchemaConflictError); + assert.equal(Buffer.compare(bytes, await readFile(path)), 0, "the file still holds the slot"); + } +}); + +test("a record that arrived under an id the reader does not list compatible is never consented to", async () => { + const path = await copyOf("extra-slot-consent.cc"); + const reader = surfaceDocument("tests.union_surface.v2"); // resolves by type name; lists nothing + const cache = CultCache.builder().withDocumentType(reader).withGenericStore(new SingleFileMessagePackBackingStore(path)).build(); + await cache.pullAllBackingStores(); + assert.equal(cache.unreadSlots(reader, "consented")?.explained, false); + assert.equal(cache.isWritable(reader, "consented"), false); +}); + +// isWritable is the write's own judge, not a second one: for every key state it answers what put does. +test("isWritable equals the outcome of put for every key state the write judges", async () => { + const typed = (type: string, schemaId: string, extra: Record = {}) => + defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: surfaceSchema, codec: surfaceCodec, name: "id", ...extra }); + const older = typed("tests.surface_older", "tests.surface_older.v1"); + const newer = typed("tests.surface_newer", "tests.surface_newer.v1", { compatibleSchemaIds: ["tests.surface_older.v1"] }); + const stranger = typed("tests.surface_stranger", "tests.surface_stranger.v1"); + const global = typed("tests.surface_global", "tests.surface_global.v1", { global: true }); + const clean = (key: string, type: string, schemaId: string) => ({ ...envelopeOf(new Uint8Array()), key, type, schemaId }); + const held = async (key: string, file: string, type: string, schemaId: string) => + ({ ...envelopeOf(await read(file)), key, type, schemaId }); + + type Row = { name: string; build: () => Promise<{ cache: CultCache; target: typeof older; key: string }> }; + const rows: Row[] = [ + { name: "owned clean record in the writer's own store", build: async () => { + const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer) + .withGenericStore(memoryStore([await held("k", "union-surface.msgpack", older.type, older.schemaId!)])).build(); + return { cache, target: newer, key: "k" }; + } }, + { name: "owned record held in a store other than the writer's home", build: async () => { + const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer) + .withBackingStore(memoryStore([await held("k", "union-surface.msgpack", older.type, older.schemaId!)]), older) + .withBackingStore(memoryStore([]), newer).build(); + return { cache, target: newer, key: "k" }; + } }, + { name: "a record of a type that does not own the writer's id", build: async () => { + const cache = CultCache.builder().withDocumentType(older).withDocumentType(stranger) + .withGenericStore(memoryStore([await held("k", "union-surface.msgpack", older.type, older.schemaId!)])).build(); + return { cache, target: stranger, key: "k" }; + } }, + { name: "a global type at a key other than the global key", build: async () => { + const cache = CultCache.builder().withDocumentType(global).withGenericStore(memoryStore([])).build(); + return { cache, target: global, key: "x" }; + } }, + { name: "a key a store carries foreign", build: async () => { + const cache = CultCache.builder().withDocumentType(older) + .withGenericStore(memoryStore([{ ...clean("k", "tests.unknown", "tests.unknown.v1"), payload: documentPayloadFromBytes(await read("union-surface.msgpack")) }])).build(); + return { cache, target: older, key: "k" }; + } }, + { name: "a marked record of the writer's own type", build: async () => { + const cache = CultCache.builder().withDocumentType(older) + .withGenericStore(memoryStore([await held("k", "extra-slot-record.msgpack", older.type, older.schemaId!)])).build(); + return { cache, target: older, key: "k" }; + } }, + { name: "an unheld key", build: async () => ({ cache: CultCache.builder().withDocumentType(older).withGenericStore(memoryStore([])).build(), target: older, key: "k" }) }, + ]; + const outcomes: [string, boolean, boolean][] = []; + for (const row of rows) { + const { cache, target, key } = await row.build(); + await cache.pullAllBackingStores(); + const predicted = cache.isWritable(target, key); + const landed = await cache.put(target, key === "x" ? "x" : key, golden).then(() => true, () => false); + assert.equal(predicted, landed, row.name); + outcomes.push([row.name, predicted, landed]); + } + assert.deepEqual(outcomes.map(([, writable]) => writable), [true, false, false, false, false, false, true]); + // The same judge refuses a removal of a key held read-only or carried foreign. + const removals = CultCache.builder().withDocumentType(older) + .withGenericStore(memoryStore([await held("k", "extra-slot-record.msgpack", older.type, older.schemaId!)])).build(); + await removals.pullAllBackingStores(); + await assert.rejects(() => removals.delete(older, "k"), SchemaConflictError); + const stranger2 = CultCache.builder().withDocumentType(older) + .withGenericStore(memoryStore([{ ...envelopeOf(await read("union-surface.msgpack")), key: "f", type: "tests.unknown", schemaId: "tests.unknown.v1" }])).build(); + await stranger2.pullAllBackingStores(); + await assert.rejects(() => stranger2.delete(older, "f"), SchemaConflictError); +}); + +// A removal is by key (C# Remove): the owner of a held record's id removes the one record that exists, in one store call and +// with the cache dropped only once the store has; a type that does not own it is refused like put, and isWritable agrees. +test("delete of a key held as another type: the owner removes it once, a stranger is refused, a failing store leaves cache and store agreeing", async () => { + const typed = (type: string, schemaId: string, extra: Record = {}) => + defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: surfaceSchema, codec: surfaceCodec, name: "id", ...extra }); + const older = typed("tests.surface_older", "tests.surface_older.v1"); + const newer = typed("tests.surface_newer", "tests.surface_newer.v1", { compatibleSchemaIds: ["tests.surface_older.v1"] }); + const stranger = typed("tests.surface_stranger", "tests.surface_stranger.v1"); + const heldRecord = async (): Promise => ({ ...envelopeOf(await read("union-surface.msgpack")), key: "k", type: older.type, schemaId: older.schemaId }); + const open = async (failing = false) => { + const rows = [await heldRecord()]; + const calls: string[] = []; + const store: CacheBackingStore = { + async pullAll() { return rows.map((row) => ({ ...row })); }, + async push(entry) { rows.push(entry); }, + async delete(entry) { + calls.push(entry.key); + if (failing) throw new Error("store refused the removal"); + rows.splice(rows.findIndex((row) => row.key === entry.key), 1); + }, + }; + const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withDocumentType(stranger).withGenericStore(store).build(); + await cache.pullAllBackingStores(); + return { cache, rows, calls }; + }; + + const owner = await open(); + assert.equal(owner.cache.isWritable(newer, "k"), true); + assert.equal(await owner.cache.delete(newer, "k"), true, "the owner's removal removes the one record"); + assert.deepEqual([owner.rows.length, owner.calls.length, owner.cache.get(older, "k")], [0, 1, undefined], "one store call, record gone from store and cache"); + + const refused = await open(); + assert.equal(refused.cache.isWritable(stranger, "k"), false); + await assert.rejects(() => refused.cache.delete(stranger, "k"), SchemaConflictError); + assert.deepEqual([refused.rows.length, refused.calls.length, refused.cache.get(older, "k") !== undefined], [1, 0, true]); + + const failing = await open(true); + await assert.rejects(() => failing.cache.delete(newer, "k"), /store refused/u); + assert.deepEqual([failing.rows.length, failing.calls.length, failing.cache.get(older, "k") !== undefined], [1, 1, true], "the store failed: neither side dropped it"); + + const same = await open(); + assert.equal(await same.cache.delete(older, "k"), true); + assert.equal(await same.cache.delete(older, "k"), false, "an unheld key is still the plain false"); +}); + +// The removal form's three finer behaviours (cultlib-gaps:finding:cut-slot-readonly-ts.s4.removal-form-unpinned). +test("an owner's delete: refused for a marked record, taken from the held record's store, and clears the held type's lookups", async () => { + const typed = (type: string, schemaId: string, extra: Record = {}) => + defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: surfaceSchema, codec: surfaceCodec, name: "id", ...extra }); + const older = typed("tests.surface_older", "tests.surface_older.v1", { indexes: { byVersion: (value: Surface) => String(value.version) } }); + const newer = typed("tests.surface_newer", "tests.surface_newer.v1", { compatibleSchemaIds: ["tests.surface_older.v1"] }); + const heldIn = async (file: string): Promise => + ({ ...envelopeOf(await read(file)), key: "k", type: older.type, schemaId: older.schemaId }); + const recording = (rows: CultCacheEnvelope[]) => { + const removed: string[] = []; + const store: CacheBackingStore = { + async pullAll() { return rows.map((row) => ({ ...row })); }, + async push(entry) { rows.push(entry); }, + async delete(entry) { removed.push(entry.key); rows.splice(rows.findIndex((row) => row.key === entry.key), 1); }, + }; + return { store, removed, rows }; + }; + + // 1. The owner may not remove a record carrying unread slots nothing consents to. + const marked = recording([await heldIn("extra-slot-record.msgpack")]); + const markedCache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(marked.store).build(); + await markedCache.pullAllBackingStores(); + await assert.rejects(() => markedCache.delete(newer, "k"), (error: unknown) => error instanceof SchemaConflictError && error.unreadSlots?.typeName === "UnionSurface"); + assert.deepEqual([marked.rows.length, marked.removed.length, markedCache.get(older, "k") !== undefined], [1, 0, true]); + + // 2. The removal goes to the store of the record that exists, not the deleting type's; there the write form is false (it would + // push to the wrong store) while the removal succeeds. + const olderStore = recording([await heldIn("union-surface.msgpack")]); + const newerStore = recording([]); + const split = CultCache.builder().withDocumentType(older).withDocumentType(newer) + .withBackingStore(olderStore.store, older).withBackingStore(newerStore.store, newer).build(); + await split.pullAllBackingStores(); + assert.equal(split.isWritable(newer, "k"), false, "isWritable answers the write form: the rename cannot replace a record in another store"); + assert.equal(await split.delete(newer, "k"), true); + assert.deepEqual([olderStore.removed, newerStore.removed, split.get(older, "k")], [["k"], [], undefined]); + + // 3. The held type's name and index lookups are cleared with the record. + const single = recording([await heldIn("union-surface.msgpack")]); + const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(single.store).build(); + await cache.pullAllBackingStores(); + assert.equal(cache.getKeyByName(older, "surface"), "k"); + assert.equal(cache.getKeyByIndex(older, "byVersion", "3"), "k"); + await cache.delete(newer, "k"); + assert.equal(cache.getKeyByName(older, "surface"), undefined); + assert.equal(cache.getKeyByIndex(older, "byVersion", "3"), undefined); + assert.equal(cache.getAll(older).length, 0); +}); + +test("a global type at a key other than the global key: isWritable is false and delete is the plain false of an unheld key", async () => { + const global = defineDocumentType({ type: "tests.surface_global", schemaId: "tests.surface_global.v1", schemaName: "tests.surface_global", schemaVersion: "tests.surface_global.v1", schema: surfaceSchema, codec: surfaceCodec, name: "id", global: true }); + const cache = CultCache.builder().withDocumentType(global).withGenericStore(memoryStore([])).build(); + assert.equal(cache.isWritable(global, "x"), false); + assert.equal(await cache.delete(global, "x"), false); +}); diff --git a/packages/cultcache-ts/test/union-parity.test.ts b/packages/cultcache-ts/test/union-parity.test.ts index 46dcb42f..d688cb74 100644 --- a/packages/cultcache-ts/test/union-parity.test.ts +++ b/packages/cultcache-ts/test/union-parity.test.ts @@ -1,8 +1,7 @@ import assert from "node:assert/strict"; -import { existsSync } from "node:fs"; import { mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; import { test } from "node:test"; import { decode, encode } from "@msgpack/msgpack"; @@ -22,190 +21,14 @@ import { encodeDocumentPayload, } from "../src/document"; import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; -import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types"; +import type { AnyCultCacheDocumentDefinition, CultCacheEnvelope } from "../src/types"; + +import { + CANARY, type Node, type Surface, golden, hex, node, outcome, read, surfaceCodec, surfaceDocument, surfaceSchema, vectors, envelopeOf, memoryStore, +} from "./union-surface-fixture"; const { array, boolean, bytes, float32, float64, int32, int64, lazy, list, nullable, slots, string, union, unit } = codecs; -// The names the C# reference gives the fixture's types (CultSchemaTypeNames.FromType), as union-surface.cc carries them. -const CS = "GameCult.Caching.Tests.UnionParityVectorTests+"; - -// The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected -// outcome is read from those committed files; only the value the payload must decode to is spelled here. -function findVectors(from: string): string { - for (let dir = from; ; dir = resolve(dir, "..")) { - if (existsSync(join(dir, "tests", "vectors", "union-parity"))) return join(dir, "tests", "vectors", "union-parity"); - if (resolve(dir, "..") === dir) throw new Error("tests/vectors/union-parity not found above the test."); - } -} -const vectors = findVectors(__dirname); -const read = async (name: string) => new Uint8Array(await readFile(join(vectors, name))); - -// The string the C# writer put inside every refusal payload, to prove a refusal never echoes the payload. -const CANARY = "CANARY-union-golden-4c2a"; - -type Node = - | { kind: "Box"; id: string; children: (Node | null)[] } - | { kind: "Text"; id: string; content: string } - | { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] } - | { kind: "Image"; id: string; uri: string; alt: string } - | { kind: "Wrap"; inner: Node | null; id: string }; -interface Composition { name: string; body: Node | null } -type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" }; -// The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records. -type GoldenOutcome = - | { kind: "Pending"; message: string; id: string } - | { kind: "Completed"; sourceVersion: number; id: string } - | { kind: "Denied"; id: string } - | { kind: "Floor"; id: string } - | { kind: "Ceiling"; id: string }; -type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; -type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; -interface Surface { - id: string; - version: number; - root: Node | null; - outcomes: (GoldenOutcome | null)[]; - part: Item | null; - crafted: CraftedItem | null; - missing: Node | null; -} - -const composition = slots({ 0: ["name", string], 1: ["body", lazy(() => node)] }, `${CS}Composition`); -const node: Codec = union("Node", { - 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })], - 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })], - 2: ["Use", slots<{ id: string; composition: Composition; slots: (Node | null)[] }>({ - 0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))], - })], - 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], - 6: ["Wrap", slots<{ inner: Node | null; id: string }>({ 0: ["inner", lazy(() => node)], 1: ["id", string] })], -}, `${CS}Node`); -const outcome = union("Outcome", { - 0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })], - 1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", int32] })], - 2: ["Denied", unit], - [-2147483648]: ["Floor", unit], - 2147483647: ["Ceiling", unit], -}, `${CS}Outcome`); -const idOnly = slots<{ id: string }>({ 0: ["id", string] }); -const goldenOutcome = union("Outcome", { - 0: ["Pending", slots<{ message: string; id: string }>({ 0: ["message", string], 1: ["id", string] })], - 1: ["Completed", slots<{ sourceVersion: number; id: string }>({ 0: ["sourceVersion", int32], 1: ["id", string] })], - 2: ["Denied", idOnly], - [-2147483648]: ["Floor", idOnly], - 2147483647: ["Ceiling", idOnly], -}, `${CS}Outcome`); -const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] }); -const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] }); -const item = union("Item", { - 0: ["Gem", slots<{ name: string }>({ 0: ["name", string] })], - 1: ["Tool", toolArm], - 2: ["Part", partArm], -}, `${CS}Item`); -const craftedItem = union("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }, `${CS}CraftedItem`); -const surfaceCodec = slots({ - 0: ["id", string], - 1: ["version", int32], - 2: ["root", node], - 3: ["outcomes", list(goldenOutcome)], - 4: ["part", item], - 5: ["crafted", craftedItem], - 6: ["missing", node, null], -}, `${CS}UnionSurface`); - -const nodeSchema: z.ZodType = z.lazy(() => - z.union([ - z.null(), - z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("Box"), id: z.string(), children: z.array(nodeSchema) }), - z.object({ kind: z.literal("Text"), id: z.string(), content: z.string() }), - z.object({ - kind: z.literal("Use"), - id: z.string(), - composition: z.object({ name: z.string(), body: nodeSchema }), - slots: z.array(nodeSchema), - }), - z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }), - z.object({ kind: z.literal("Wrap"), inner: nodeSchema, id: z.string() }), - ]), - ])); -const surfaceSchema = z.object({ - id: z.string(), - version: z.number(), - root: nodeSchema, - outcomes: z.array(z.union([ - z.null(), - z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("Pending"), message: z.string(), id: z.string() }), - z.object({ kind: z.literal("Completed"), sourceVersion: z.number(), id: z.string() }), - z.object({ kind: z.literal("Denied"), id: z.string() }), - z.object({ kind: z.literal("Floor"), id: z.string() }), - z.object({ kind: z.literal("Ceiling"), id: z.string() }), - ]), - ])), - part: z.union([z.null(), z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("Gem"), name: z.string() }), - z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }), - z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }), - ])]), - crafted: z.union([z.null(), z.discriminatedUnion("kind", [ - z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }), - z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }), - ])]), - missing: nodeSchema, -}) as unknown as z.ZodType; - -function surfaceDocument(schemaId?: string) { - return defineDocumentType({ - type: "tests.union_surface", - schemaId: schemaId ?? "tests.union_surface.v1", - schemaName: "tests.union_surface", - schemaVersion: "tests.union_surface.v1", - schema: surfaceSchema, - codec: surfaceCodec, - name: "id", - }); -} - -const golden: Surface = { - id: "surface", - version: 3, - root: { - kind: "Box", - id: "0000000000c0", - children: [ - { kind: "Text", id: "0000000000c1", content: "hello" }, - { - kind: "Use", - id: "0000000000c2", - composition: { - name: "comp", - body: { - kind: "Box", - id: "0000000000d0", - children: [ - { kind: "Text", id: "0000000000d1", content: "x" }, - { kind: "Image", id: "0000000000d2", uri: "uri://a", alt: "alt" }, - ], - }, - }, - slots: [{ kind: "Text", id: "0000000000e1", content: "slot" }, null, { kind: "Box", id: "0000000000e2", children: [] }], - }, - { kind: "Image", id: "0000000000c3", uri: "uri://b", alt: "" }, - ], - }, - outcomes: [ - { kind: "Pending", message: "wait", id: "0000000000a1" }, - { kind: "Completed", sourceVersion: 7, id: "0000000000a2" }, - { kind: "Denied", id: "0000000000a3" }, - ], - part: { kind: "Tool", name: "hammer", uses: 12 }, - crafted: { kind: "Part", name: "gear", count: 4 }, - missing: null, -}; - -const hex = (data: Uint8Array) => Buffer.from(data).toString("hex"); - async function outcomeEntries() { const entries = decode(await read("outcomes.msgpack")) as [string, string, string, number | null, string][]; return entries.map(([file, outcome, unionName, key, detail]) => ({ file, outcome, unionName, key, detail })); @@ -729,19 +552,6 @@ test("the Int32 edges of an arm key: the largest is an arm, one past either end assert.throws(() => edge.decode(readWire(fromHex("92d3ffffffff8000000090"))), { message: "Union U: unknown arm key -2147483648 (declared keys: 2147483647)." }); }); -function envelopeOf(payload: Uint8Array): CultCacheEnvelope { - return { key: "surface", type: "tests.union_surface", payload: documentPayloadFromBytes(payload), storedAt: new Date().toISOString() }; -} - -function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } { - return { - entries, - async pullAll() { return entries.map((entry) => ({ ...entry })); }, - async push(entry) { entries.push(entry); }, - async delete() {}, - }; -} - test("arm keys span Int32 and are listed ascending; slot keys are non-negative Int32", () => { const keys = union<{ kind: "High" | "Zero" | "Neg" | "Low" }>("U", { 2147483647: ["High", unit], diff --git a/packages/cultcache-ts/test/union-surface-fixture.ts b/packages/cultcache-ts/test/union-surface-fixture.ts new file mode 100644 index 00000000..1b556eb1 --- /dev/null +++ b/packages/cultcache-ts/test/union-surface-fixture.ts @@ -0,0 +1,240 @@ +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { z } from "zod"; + +import { codecs } from "../src"; +import type { Codec } from "../src/codec"; +import { defineDocumentType, documentPayloadFromBytes } from "../src/document"; +import type { CacheBackingStore, CultCacheEnvelope } from "../src/types"; + +// The union-surface fixture: the codecs, schema and value tests/vectors/union-parity was written from, shared by the +// union-parity walker and the slot-readonly walker so the shape is spelled once. +const { list, int32, lazy, slots, string, union, unit } = codecs; + + +// The names the C# reference gives the fixture's types (CultSchemaTypeNames.FromType), as union-surface.cc carries them. +export const CS = "GameCult.Caching.Tests.UnionParityVectorTests+"; + +// The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected +// outcome is read from those committed files; only the value the payload must decode to is spelled here. +export function findVectors(from: string): string { + for (let dir = from; ; dir = resolve(dir, "..")) { + if (existsSync(join(dir, "tests", "vectors", "union-parity"))) return join(dir, "tests", "vectors", "union-parity"); + if (resolve(dir, "..") === dir) throw new Error("tests/vectors/union-parity not found above the test."); + } +} +export const vectors = findVectors(__dirname); +export const read = async (name: string) => new Uint8Array(await readFile(join(vectors, name))); + +// The string the C# writer put inside every refusal payload, to prove a refusal never echoes the payload. +export const CANARY = "CANARY-union-golden-4c2a"; + +export type Node = + | { kind: "Box"; id: string; children: (Node | null)[] } + | { kind: "Text"; id: string; content: string } + | { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] } + | { kind: "Image"; id: string; uri: string; alt: string } + | { kind: "Wrap"; inner: Node | null; id: string }; +export interface Composition { name: string; body: Node | null } +export type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" }; +// The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records. +export type GoldenOutcome = + | { kind: "Pending"; message: string; id: string } + | { kind: "Completed"; sourceVersion: number; id: string } + | { kind: "Denied"; id: string } + | { kind: "Floor"; id: string } + | { kind: "Ceiling"; id: string }; +export type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; +export type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number }; +export interface Surface { + id: string; + version: number; + root: Node | null; + outcomes: (GoldenOutcome | null)[]; + part: Item | null; + crafted: CraftedItem | null; + missing: Node | null; +} + +export const composition = slots({ 0: ["name", string], 1: ["body", lazy(() => node)] }, `${CS}Composition`); +export const node: Codec = union("Node", { + 0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] }, `${CS}Box`)], + 1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] }, `${CS}Text`)], + 2: ["Use", slots<{ id: string; composition: Composition; slots: (Node | null)[] }>({ + 0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))], + })], + 5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })], + 6: ["Wrap", slots<{ inner: Node | null; id: string }>({ 0: ["inner", lazy(() => node)], 1: ["id", string] })], +}, `${CS}Node`); +export const outcome = union("Outcome", { + 0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })], + 1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", int32] })], + 2: ["Denied", unit], + [-2147483648]: ["Floor", unit], + 2147483647: ["Ceiling", unit], +}, `${CS}Outcome`); +export const idOnly = slots<{ id: string }>({ 0: ["id", string] }); +export const goldenOutcome = union("Outcome", { + 0: ["Pending", slots<{ message: string; id: string }>({ 0: ["message", string], 1: ["id", string] })], + 1: ["Completed", slots<{ sourceVersion: number; id: string }>({ 0: ["sourceVersion", int32], 1: ["id", string] })], + 2: ["Denied", idOnly], + [-2147483648]: ["Floor", idOnly], + 2147483647: ["Ceiling", idOnly], +}, `${CS}Outcome`); +export const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] }); +export const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] }); +export const item = union("Item", { + 0: ["Gem", slots<{ name: string }>({ 0: ["name", string] })], + 1: ["Tool", toolArm], + 2: ["Part", partArm], +}, `${CS}Item`); +export const craftedItem = union("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }, `${CS}CraftedItem`); +export const surfaceCodecOf = (typeName: string) => slots({ + 0: ["id", string], + 1: ["version", int32], + 2: ["root", node], + 3: ["outcomes", list(goldenOutcome)], + 4: ["part", item], + 5: ["crafted", craftedItem], + 6: ["missing", node, null], +}, typeName); +export const surfaceCodec = surfaceCodecOf(`${CS}UnionSurface`); + +export const nodeSchema: z.ZodType = z.lazy(() => + z.union([ + z.null(), + z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Box"), id: z.string(), children: z.array(nodeSchema) }), + z.object({ kind: z.literal("Text"), id: z.string(), content: z.string() }), + z.object({ + kind: z.literal("Use"), + id: z.string(), + composition: z.object({ name: z.string(), body: nodeSchema }), + slots: z.array(nodeSchema), + }), + z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }), + z.object({ kind: z.literal("Wrap"), inner: nodeSchema, id: z.string() }), + ]), + ])); +export const surfaceSchema = z.object({ + id: z.string(), + version: z.number(), + root: nodeSchema, + outcomes: z.array(z.union([ + z.null(), + z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Pending"), message: z.string(), id: z.string() }), + z.object({ kind: z.literal("Completed"), sourceVersion: z.number(), id: z.string() }), + z.object({ kind: z.literal("Denied"), id: z.string() }), + z.object({ kind: z.literal("Floor"), id: z.string() }), + z.object({ kind: z.literal("Ceiling"), id: z.string() }), + ]), + ])), + part: z.union([z.null(), z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Gem"), name: z.string() }), + z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }), + z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }), + ])]), + crafted: z.union([z.null(), z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }), + z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }), + ])]), + missing: nodeSchema, +}) as unknown as z.ZodType; + +// The reference's SurfaceDeclaring: the same layout, schema tests.union_surface.v2, listing the older wide id compatible. +export function surfaceDeclaringDocument() { + return defineDocumentType({ + type: "tests.union_surface", + schemaId: "tests.union_surface.v2", + schemaName: "tests.union_surface", + schemaVersion: "tests.union_surface.v2", + compatibleSchemaIds: ["tests.union_surface.wide"], + schema: surfaceSchema, + codec: surfaceCodecOf(`${CS}SurfaceDeclaring`), + name: "id", + }); +} + +export function surfaceDocument(schemaId?: string, contentHash?: string) { + return defineDocumentType({ + ...(contentHash === undefined ? {} : { contentHash }), + type: "tests.union_surface", + schemaId: schemaId ?? "tests.union_surface.v1", + schemaName: "tests.union_surface", + schemaVersion: "tests.union_surface.v1", + schema: surfaceSchema, + codec: surfaceCodec, + name: "id", + }); +} + +export const golden: Surface = { + id: "surface", + version: 3, + root: { + kind: "Box", + id: "0000000000c0", + children: [ + { kind: "Text", id: "0000000000c1", content: "hello" }, + { + kind: "Use", + id: "0000000000c2", + composition: { + name: "comp", + body: { + kind: "Box", + id: "0000000000d0", + children: [ + { kind: "Text", id: "0000000000d1", content: "x" }, + { kind: "Image", id: "0000000000d2", uri: "uri://a", alt: "alt" }, + ], + }, + }, + slots: [{ kind: "Text", id: "0000000000e1", content: "slot" }, null, { kind: "Box", id: "0000000000e2", children: [] }], + }, + { kind: "Image", id: "0000000000c3", uri: "uri://b", alt: "" }, + ], + }, + outcomes: [ + { kind: "Pending", message: "wait", id: "0000000000a1" }, + { kind: "Completed", sourceVersion: 7, id: "0000000000a2" }, + { kind: "Denied", id: "0000000000a3" }, + ], + part: { kind: "Tool", name: "hammer", uses: 12 }, + crafted: { kind: "Part", name: "gear", count: 4 }, + missing: null, +}; +export const hex = (data: Uint8Array) => Buffer.from(data).toString("hex"); + +// tests.slot_unit: a document [Name, Outcome: UnitOutcome], whose one arm Denied2 is a unit arm (extra-slot-store.cc, record +// unit-arm). The reader's names are the reference's: a mark names Denied2. +export const unitOutcome = union<{ kind: "Denied2" }>("UnitOutcome", { 2: ["Denied2", unit] }, `${CS}UnitOutcome`); +export function slotUnitDocument(schemaId: string, contentHash?: string) { + return defineDocumentType({ + ...(contentHash === undefined ? {} : { contentHash }), + type: "tests.slot_unit", + schemaId, + schemaName: "tests.slot_unit", + schemaVersion: "tests.slot_unit.v1", + schema: z.object({ name: z.string(), outcome: z.union([z.null(), z.object({ kind: z.literal("Denied2") })]) }), + codec: slots<{ name: string; outcome: { kind: "Denied2" } | null }>({ 0: ["name", string], 1: ["outcome", unitOutcome] }, `${CS}SlotUnit`), + name: "name", + }); +} + +export function envelopeOf(payload: Uint8Array): CultCacheEnvelope { + return { key: "surface", type: "tests.union_surface", payload: documentPayloadFromBytes(payload), storedAt: new Date().toISOString() }; +} + +export function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } { + return { + entries, + async pullAll() { return entries.map((entry) => ({ ...entry })); }, + async push(entry) { entries.push(entry); }, + async delete() {}, + }; +} +