cultcache-py: the MessagePack codec owns map keys and 64-bit integers both ways, one default author, payload-free refusals, atomic migrate_payloads
Fix batch h2 for cut py-msgpack-default (r2). The codec's read hook (_checked_map) and write check carry the C# reference's key rule; PayloadCodecFault and a coded CultCachePayloadFormatError replace the map_keys walk and the JSON remedy sentence; refusals are raised after the except block so no decoder exception rides on __context__. migrate_payloads rewrites a 0.3 store whole or not at all. The store's schema-stamp recovery and the optional-msgpack guards are deleted. Not yet run: this commit's tests are verified by the next commit's job. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
7f2c747743
commit
ed0dc50243
9 changed files with 558 additions and 180 deletions
|
|
@ -119,16 +119,26 @@ A definition's `payload_codec` owns its payload bytes. The default,
|
|||
so the registered definition decides how every record is read, and a record in
|
||||
another format is refused with `CultCachePayloadFormatError`, never guessed at.
|
||||
|
||||
- Map keys are `str` or `bytes`; any other key type is refused on write, because
|
||||
the reader refuses it. Integers beyond 64 bits are refused on write.
|
||||
- A store written by cultcache-py 0.3 or earlier holds JSON payloads. Read it by
|
||||
defining the type with `payload_codec=JSON_PAYLOAD_CODEC`, then put each value
|
||||
under the default definition to rewrite the store as MessagePack.
|
||||
- Map keys are `str`, integer, float, `bool` or `msgpack.Timestamp`, as the C#
|
||||
reference's object reader takes them; `None`, `bytes`, list and map keys are
|
||||
refused, on read and on write. Two keys Python holds as one are refused on read
|
||||
as `duplicate_map_key`, which includes `1` beside `1.0` or `True`: the reference
|
||||
keeps those apart and a Python `dict` cannot.
|
||||
- Integers beyond 64 bits (outside `-(2**63)..2**64-1`) are refused on write as
|
||||
`integer_out_of_range`; MessagePack has no wider integer.
|
||||
- A store written by cultcache-py 0.3 or earlier holds JSON payloads. Define each
|
||||
type with `payload_codec=JSON_PAYLOAD_CODEC` to read it, or rewrite it once,
|
||||
whole or not at all, with `migrate_payloads(store, definitions)`: it reads
|
||||
every record of the listed definitions, writes them under the definitions'
|
||||
own codec in one `push_all`, and raises `CultCacheError` naming the type,
|
||||
record key and code, with the store untouched, if any record is refused. A
|
||||
second run is refused and changes nothing.
|
||||
- A JSON payload that is a single digit (`0` to `9`) is also a valid one-byte
|
||||
MessagePack integer and reads as 48 to 57 under the default. Documents that
|
||||
are objects or lists never meet this.
|
||||
- A `PayloadCodec` you build owns the nesting depth of nodes the library's walk
|
||||
cannot see (anything that is not a mapping, sequence or set).
|
||||
cannot see (anything that is not a mapping, sequence or set), and may raise
|
||||
`PayloadCodecFault(code, kind)` for a key or integer its format cannot carry.
|
||||
|
||||
## DatabaseEntry Slot Contract
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from .backing_store import BackingStore, CultCacheEnvelope
|
||||
from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember
|
||||
from . import leaves
|
||||
from .cache import CultCache, CultCacheBuilder
|
||||
from .cache import CultCache, CultCacheBuilder, migrate_payloads
|
||||
from .documents import (
|
||||
MAX_PAYLOAD_DEPTH,
|
||||
JSON_PAYLOAD_CODEC,
|
||||
|
|
@ -12,6 +12,7 @@ from .documents import (
|
|||
DatabaseEntryField,
|
||||
DocumentDefinition,
|
||||
PayloadCodec,
|
||||
PayloadCodecFault,
|
||||
UnionArm,
|
||||
database_entry_field,
|
||||
define_database_entry_type,
|
||||
|
|
@ -40,6 +41,7 @@ __all__ = [
|
|||
"MAX_SLOT_KEY",
|
||||
"MESSAGEPACK_PAYLOAD_CODEC",
|
||||
"PayloadCodec",
|
||||
"PayloadCodecFault",
|
||||
"SingleFileMessagePackBackingStore",
|
||||
"SlotCodec",
|
||||
"UnionArm",
|
||||
|
|
@ -50,4 +52,5 @@ __all__ = [
|
|||
"define_document_type",
|
||||
"define_record_codec",
|
||||
"define_union",
|
||||
"migrate_payloads",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@
|
|||
import functools
|
||||
import threading
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any, Callable, Generic, TypeVar
|
||||
from typing import Any, Callable, Generic, Iterable, TypeVar
|
||||
|
||||
from .backing_store import BackingStore, CultCacheEnvelope
|
||||
from .documents import DocumentDefinition, extract_value
|
||||
from .documents import JSON_PAYLOAD_CODEC, DocumentDefinition, PayloadCodec, extract_value
|
||||
|
||||
T = TypeVar("T")
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
|
@ -585,3 +585,40 @@ class CultCache:
|
|||
if document is not None:
|
||||
return document
|
||||
return self._state.documents_by_schema_name.get(envelope.type)
|
||||
|
||||
|
||||
def migrate_payloads(
|
||||
store: BackingStore,
|
||||
definitions: Iterable[DocumentDefinition[Any]],
|
||||
*,
|
||||
source_codec: PayloadCodec = JSON_PAYLOAD_CODEC,
|
||||
) -> int:
|
||||
"""Rewrite the records of `definitions` in `store` from `source_codec` to each definition's own codec, whole or not
|
||||
at all: every record is read and rewritten first, any refusal raises `CultCacheError` naming the type, the record key
|
||||
and the refusal's code before the store is touched, and the rewritten envelopes go back in one `push_all`. Records of
|
||||
other types keep their bytes. Returns the count rewritten.
|
||||
|
||||
Run it once on a store written by cultcache-py 0.3 or earlier. A store already in the definitions' format is
|
||||
refused, not rewritten twice: the second run raises and changes nothing.
|
||||
"""
|
||||
by_type: dict[str, DocumentDefinition[Any]] = {}
|
||||
for definition in definitions:
|
||||
by_type[definition.type] = definition
|
||||
by_type.setdefault(definition.catalog_entry().schema_name, definition)
|
||||
rewritten: list[CultCacheEnvelope] = []
|
||||
for envelope in store.pull_all():
|
||||
definition = by_type.get(envelope.type)
|
||||
if definition is None:
|
||||
continue
|
||||
try:
|
||||
value = replace(definition, payload_codec=source_codec).decode_payload(envelope.payload)
|
||||
rewritten.append(replace(envelope, payload=definition.encode_payload(value)))
|
||||
except ValueError as refusal:
|
||||
code = getattr(refusal, "code", type(refusal).__name__)
|
||||
failure = CultCacheError(f"Cannot migrate {envelope.type} record {envelope.key}: {code}; the store is unchanged.")
|
||||
else:
|
||||
continue
|
||||
raise failure
|
||||
if rewritten:
|
||||
store.push_all(rewritten)
|
||||
return len(rewritten)
|
||||
|
|
|
|||
|
|
@ -42,52 +42,52 @@ class CultCachePayloadDepthError(ValueError):
|
|||
)
|
||||
|
||||
|
||||
class CultCachePayloadFormatError(ValueError):
|
||||
"""A payload is not one value of its definition's codec (read), or holds a map key the codec cannot write (write).
|
||||
Names the type, the codec, the direction and the fault's class or the key's type; never a payload byte or value."""
|
||||
class PayloadCodecFault(ValueError):
|
||||
"""A codec raises this for a payload its format cannot carry. `code` names the rule (the library's are
|
||||
`map_key_kind`, `duplicate_map_key` and `integer_out_of_range`) and `kind` the kind of node at fault (a MessagePack
|
||||
kind such as `nil` or `bin`); neither is ever a payload value. A consumer-built codec may raise it."""
|
||||
|
||||
def __init__(self, type_name: str, codec: str, direction: str, message: str) -> None:
|
||||
def __init__(self, code: str, kind: str = "") -> None:
|
||||
self.code = code
|
||||
self.kind = kind
|
||||
super().__init__(code)
|
||||
|
||||
|
||||
class CultCachePayloadFormatError(ValueError):
|
||||
"""A payload is not one value of its definition's codec (read), or holds a key or integer the codec cannot carry.
|
||||
Names the type, the codec, the direction and a `code`; `detail` is the fault's class or the key's kind. Never a
|
||||
payload byte or value."""
|
||||
|
||||
def __init__(self, type_name: str, codec: str, direction: str, code: str, detail: str) -> None:
|
||||
self.type_name = type_name
|
||||
self.codec = codec
|
||||
self.direction = direction
|
||||
self.code = code
|
||||
if code == "not_one_value":
|
||||
message = f"{type_name} payload is not one {codec} value ({detail}); refused on read."
|
||||
elif code == "map_key_kind":
|
||||
message = (
|
||||
f"{type_name} payload has a map key of kind {detail}; "
|
||||
f"{codec} payloads take str, int, float, bool or timestamp keys (refused on {direction})."
|
||||
)
|
||||
elif code == "duplicate_map_key":
|
||||
message = f"{type_name} payload has two map keys Python holds as one; refused on read."
|
||||
elif code == "integer_out_of_range":
|
||||
message = f"{type_name} payload has an integer beyond 64 bits; refused on write."
|
||||
else:
|
||||
message = f"{type_name} payload is refused by the {codec} codec ({code}); refused on {direction}."
|
||||
super().__init__(message)
|
||||
|
||||
@classmethod
|
||||
def unreadable(cls, type_name: str, codec: "PayloadCodec", fault: BaseException) -> "CultCachePayloadFormatError":
|
||||
message = f"{type_name} payload is not one {codec.name} value ({type(fault).__name__}); refused on read."
|
||||
if codec is MESSAGEPACK_PAYLOAD_CODEC:
|
||||
message += (
|
||||
" A record written by cultcache-py 0.3 or earlier through define_document_type's JSON default is read by"
|
||||
f" defining {type_name} with payload_codec=JSON_PAYLOAD_CODEC."
|
||||
)
|
||||
return cls(type_name, codec.name, "read", message)
|
||||
|
||||
@classmethod
|
||||
def unwritable_key(cls, type_name: str, codec: "PayloadCodec", key: Any) -> "CultCachePayloadFormatError":
|
||||
allowed = " or ".join(kind.__name__ for kind in codec.map_keys or ())
|
||||
message = (
|
||||
f"{type_name} payload has a map key of type {type(key).__name__}; "
|
||||
f"{codec.name} payloads take {allowed} keys (refused on write)."
|
||||
)
|
||||
return cls(type_name, codec.name, "write", message)
|
||||
|
||||
|
||||
def _check_payload_tree(tree: Any, type_name: str, direction: str, codec: "PayloadCodec") -> None:
|
||||
"""The one walk over a payload tree. Iterative (every mapping, sequence and set that is not text or bytes is a
|
||||
container): refuses the first container past `MAX_PAYLOAD_DEPTH`, and, on write, a map key the codec cannot write."""
|
||||
def _check_payload_tree(tree: Any, type_name: str, direction: str) -> None:
|
||||
"""The one depth walk over a payload tree, for every codec. Iterative (every mapping, sequence and set that is not
|
||||
text or bytes is a container): refuses the first container past `MAX_PAYLOAD_DEPTH`."""
|
||||
stack = [(tree, 1)] if _is_container(tree) else []
|
||||
while stack:
|
||||
node, depth = stack.pop()
|
||||
if depth > MAX_PAYLOAD_DEPTH:
|
||||
raise CultCachePayloadDepthError(type_name, direction)
|
||||
if isinstance(node, Mapping):
|
||||
if direction == "write" and codec.map_keys is not None:
|
||||
for key in node:
|
||||
if not isinstance(key, codec.map_keys):
|
||||
raise CultCachePayloadFormatError.unwritable_key(type_name, codec, key)
|
||||
children = chain(node.keys(), node.values())
|
||||
else:
|
||||
children = node
|
||||
children = chain(node.keys(), node.values()) if isinstance(node, Mapping) else node
|
||||
stack.extend((child, depth + 1) for child in children if _is_container(child))
|
||||
|
||||
|
||||
|
|
@ -139,18 +139,73 @@ def _pack_tree(tree: Any) -> bytes:
|
|||
@dataclass(frozen=True)
|
||||
class PayloadCodec:
|
||||
"""A codec owns one payload format: how a tree becomes bytes and back, which of its decode faults mean the payload
|
||||
nests past the interpreter's own stack, and which map keys it can write. A consumer-built codec owns the depth of
|
||||
nodes the depth walk cannot see (anything that is not a Mapping, Sequence or Set)."""
|
||||
nests past the interpreter's own stack, and which trees it can carry: it raises `PayloadCodecFault` for a key or an
|
||||
integer its format cannot hold, and nothing it writes is refused by its own `decode`. A consumer-built codec owns
|
||||
the depth of nodes the depth walk cannot see (anything that is not a Mapping, Sequence or Set)."""
|
||||
|
||||
name: str
|
||||
encode: Callable[[Any], bytes]
|
||||
decode: Callable[[bytes], Any]
|
||||
depth_faults: tuple[type[BaseException], ...] = ()
|
||||
map_keys: tuple[type, ...] | None = None
|
||||
|
||||
|
||||
_INTEGER_RANGE = (-(2**63), 2**64 - 1)
|
||||
_KEY_TYPES = (str, int, float, Float32, msgpack.Timestamp)
|
||||
|
||||
|
||||
def _node_kind(node: Any) -> str:
|
||||
"""The MessagePack kind of a node, for a refusal: a kind, never a value."""
|
||||
if node is None:
|
||||
return "nil"
|
||||
if isinstance(node, (bytes, bytearray, memoryview)):
|
||||
return "bin"
|
||||
if isinstance(node, Mapping):
|
||||
return "map"
|
||||
if isinstance(node, msgpack.ExtType):
|
||||
return "ext"
|
||||
if isinstance(node, _CONTAINERS):
|
||||
return "array"
|
||||
return "unsupported"
|
||||
|
||||
|
||||
def _checked_map(pairs: list[tuple[Any, Any]]) -> dict[Any, Any]:
|
||||
"""The read hook of every MessagePack map: the keys the C# reference's object reader accepts (str, int, float,
|
||||
bool, timestamp) and no duplicate."""
|
||||
for key, _ in pairs:
|
||||
if not isinstance(key, _KEY_TYPES):
|
||||
raise PayloadCodecFault("map_key_kind", _node_kind(key))
|
||||
held = dict(pairs)
|
||||
if len(held) != len(pairs):
|
||||
raise PayloadCodecFault("duplicate_map_key")
|
||||
return held
|
||||
|
||||
|
||||
def _check_writable(tree: Any) -> None:
|
||||
"""The write side of the key and integer rule: what `_checked_map` refuses on read is refused here, so nothing
|
||||
this codec writes is refused by its own decode. Iterative; it stops at the depth limit, which the definition rules."""
|
||||
stack = [(tree, 1)]
|
||||
while stack:
|
||||
node, depth = stack.pop()
|
||||
if depth > MAX_PAYLOAD_DEPTH + 1:
|
||||
raise RecursionError("payload nests past the depth limit")
|
||||
if isinstance(node, int) and not _INTEGER_RANGE[0] <= node <= _INTEGER_RANGE[1]:
|
||||
raise PayloadCodecFault("integer_out_of_range")
|
||||
if isinstance(node, Mapping):
|
||||
for key in node:
|
||||
if not isinstance(key, _KEY_TYPES):
|
||||
raise PayloadCodecFault("map_key_kind", _node_kind(key))
|
||||
stack.extend((child, depth + 1) for child in chain(node.keys(), node.values()))
|
||||
elif isinstance(node, (list, tuple)):
|
||||
stack.extend((child, depth + 1) for child in node)
|
||||
|
||||
|
||||
def _unpack_tree(payload: bytes) -> Any:
|
||||
return msgpack.unpackb(payload, raw=False)
|
||||
return msgpack.unpackb(payload, raw=False, strict_map_key=False, object_pairs_hook=_checked_map)
|
||||
|
||||
|
||||
def _pack_checked_tree(tree: Any) -> bytes:
|
||||
_check_writable(tree)
|
||||
return _pack_tree(tree)
|
||||
|
||||
|
||||
def _dump_json(tree: Any) -> bytes:
|
||||
|
|
@ -162,10 +217,11 @@ def _load_json(payload: bytes) -> Any:
|
|||
|
||||
|
||||
MESSAGEPACK_PAYLOAD_CODEC = PayloadCodec(
|
||||
"MessagePack", _pack_tree, _unpack_tree, depth_faults=(msgpack.exceptions.StackError,), map_keys=(str, bytes)
|
||||
"MessagePack", _pack_checked_tree, _unpack_tree, depth_faults=(msgpack.exceptions.StackError,)
|
||||
)
|
||||
"""The default payload format. `map_keys` is what `unpackb` (strict_map_key) reads back, so the default never writes a
|
||||
record it cannot read."""
|
||||
"""The default payload format. It carries what the C# reference's object reader accepts: str, int, float, bool and
|
||||
timestamp keys, no duplicate key, and integers within -(2**63)..2**64-1. It refuses the rest on read through its map
|
||||
hook and on write through its encode, so it never writes a record it cannot read."""
|
||||
|
||||
JSON_PAYLOAD_CODEC = PayloadCodec("JSON", _dump_json, _load_json, depth_faults=(RecursionError,))
|
||||
"""The 0.3 default, kept for stores written before MessagePack: it writes exactly the 0.3 bytes. Never a default."""
|
||||
|
|
@ -205,6 +261,10 @@ class UnionArm:
|
|||
value: Any = None
|
||||
|
||||
|
||||
def _fault_refusal(type_name: str, codec: PayloadCodec, direction: str, fault: PayloadCodecFault) -> CultCachePayloadFormatError:
|
||||
return CultCachePayloadFormatError(type_name, codec.name, direction, fault.code, fault.kind)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentDefinition(Generic[T]):
|
||||
type: str
|
||||
|
|
@ -232,22 +292,31 @@ class DocumentDefinition(Generic[T]):
|
|||
if self.validate:
|
||||
self.validate(value)
|
||||
tree = self.encode(value)
|
||||
_check_payload_tree(tree, self.type, "write", self.payload_codec)
|
||||
return self.payload_codec.encode(tree)
|
||||
_check_payload_tree(tree, self.type, "write")
|
||||
codec = self.payload_codec
|
||||
try:
|
||||
return codec.encode(tree)
|
||||
except PayloadCodecFault as fault:
|
||||
refusal = _fault_refusal(self.type, codec, "write", fault)
|
||||
raise refusal
|
||||
|
||||
def decode_payload(self, payload: bytes) -> T:
|
||||
codec = self.payload_codec
|
||||
try:
|
||||
tree = codec.decode(payload)
|
||||
except codec.depth_faults:
|
||||
raise CultCachePayloadDepthError(self.type, "read") from None
|
||||
refusal: Exception = CultCachePayloadDepthError(self.type, "read")
|
||||
except PayloadCodecFault as fault:
|
||||
refusal = _fault_refusal(self.type, codec, "read", fault)
|
||||
except ValueError as fault:
|
||||
raise CultCachePayloadFormatError.unreadable(self.type, codec, fault) from None
|
||||
_check_payload_tree(tree, self.type, "read", codec)
|
||||
value = self.decode(tree)
|
||||
if self.validate:
|
||||
self.validate(value)
|
||||
return value
|
||||
refusal = CultCachePayloadFormatError(self.type, codec.name, "read", "not_one_value", type(fault).__name__)
|
||||
else:
|
||||
_check_payload_tree(tree, self.type, "read")
|
||||
value = self.decode(tree)
|
||||
if self.validate:
|
||||
self.validate(value)
|
||||
return value
|
||||
raise refusal
|
||||
|
||||
def catalog_entry(self) -> CultCacheSchemaCatalogEntry:
|
||||
if self._catalog_entry is not None:
|
||||
|
|
@ -300,7 +369,7 @@ def define_document_type(
|
|||
global_document: bool = False,
|
||||
name: str | Extractor[T] | None = None,
|
||||
indexes: dict[str, str | Extractor[T]] | None = None,
|
||||
payload_codec: PayloadCodec = MESSAGEPACK_PAYLOAD_CODEC,
|
||||
payload_codec: PayloadCodec | None = None,
|
||||
schema_id: str | None = None,
|
||||
schema_name: str | None = None,
|
||||
schema_version: str | None = None,
|
||||
|
|
@ -310,7 +379,9 @@ def define_document_type(
|
|||
members: Iterable[CultCacheSchemaCatalogMember] | Callable[[], tuple[CultCacheSchemaCatalogMember, ...]] | None = None,
|
||||
) -> DocumentDefinition[T]:
|
||||
"""Define a document type. Its payload is the MessagePack of what `encode` returns; pass
|
||||
`payload_codec=JSON_PAYLOAD_CODEC` to read and rewrite a store written by cultcache-py 0.3 or earlier."""
|
||||
`payload_codec=JSON_PAYLOAD_CODEC` to read a store written by cultcache-py 0.3 or earlier, and `migrate_payloads` to
|
||||
rewrite it."""
|
||||
codec = {} if payload_codec is None else {"payload_codec": payload_codec}
|
||||
return DocumentDefinition(
|
||||
type=type,
|
||||
encode=encode,
|
||||
|
|
@ -319,7 +390,7 @@ def define_document_type(
|
|||
global_document=global_document,
|
||||
name=name,
|
||||
indexes=indexes or {},
|
||||
payload_codec=payload_codec,
|
||||
**codec,
|
||||
schema_id=schema_id,
|
||||
schema_name=schema_name,
|
||||
schema_version=schema_version,
|
||||
|
|
|
|||
|
|
@ -16,10 +16,7 @@ from collections.abc import Mapping
|
|||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, NoReturn
|
||||
|
||||
try:
|
||||
from msgpack import ExtType as _ExtType
|
||||
except ModuleNotFoundError:
|
||||
_ExtType = () # type: ignore[assignment,misc]
|
||||
from msgpack import ExtType as _ExtType
|
||||
|
||||
_bool, _bytes, _list, _int, _float, _str = bool, bytes, list, int, float, str
|
||||
_BYTE_LIKE = (_bytes, bytearray, memoryview)
|
||||
|
|
|
|||
|
|
@ -214,19 +214,7 @@ def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None:
|
|||
raise ValueError("CultCache persisted records must declare storedAt")
|
||||
catalog_entry = catalog_by_schema_id.get(schema_id)
|
||||
if catalog_entry is None:
|
||||
schema_version = _infer_schema_version_from_payload(_normalize_payload(payload))
|
||||
schema_name = _infer_schema_name(schema_version) if schema_version is not None else None
|
||||
if schema_name is None:
|
||||
raise ValueError(f'CultCache persisted record "{key}" references missing schema id "{schema_id}"')
|
||||
catalog_entry = CultCacheSchemaCatalogEntry(
|
||||
schema_id=schema_id,
|
||||
schema_name=schema_name,
|
||||
schema_version=schema_version,
|
||||
content_hash=schema_id,
|
||||
canonical_schema_json="",
|
||||
compatible_schema_ids=(),
|
||||
members=(),
|
||||
)
|
||||
raise ValueError(f'CultCache persisted record "{key}" references missing schema id "{schema_id}"')
|
||||
normalized_payload = _normalize_payload(payload)
|
||||
envelopes.append(
|
||||
CultCacheEnvelope(
|
||||
|
|
@ -242,6 +230,9 @@ def _decode_v1_snapshot(decoded: Any) -> list[CultCacheEnvelope] | None:
|
|||
|
||||
|
||||
def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]:
|
||||
"""A legacy envelope holds its payload as an inline MessagePack subtree or as bytes. Extracting the subtree's bytes
|
||||
decides no format: the inline value is MessagePack by construction, and the registered codec still reads it or
|
||||
refuses it."""
|
||||
if not isinstance(decoded, list):
|
||||
raise ValueError("CultCache MessagePack store is not a recognized snapshot")
|
||||
envelopes: list[CultCacheEnvelope] = []
|
||||
|
|
@ -365,26 +356,3 @@ def _normalize_payload(payload: Any) -> bytes:
|
|||
if isinstance(payload, list) and all(isinstance(item, int) and 0 <= item <= 255 for item in payload):
|
||||
return bytes(payload)
|
||||
raise ValueError("CultCache record payload must be binary MessagePack bytes")
|
||||
|
||||
|
||||
def _infer_schema_version_from_payload(payload: bytes) -> str | None:
|
||||
try:
|
||||
decoded = msgpack.unpackb(payload, raw=False)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(decoded, list) and decoded and isinstance(decoded[0], str):
|
||||
return decoded[0]
|
||||
if isinstance(decoded, dict):
|
||||
value = decoded.get("schemaVersion", decoded.get("schema_version"))
|
||||
return value if isinstance(value, str) else None
|
||||
return None
|
||||
|
||||
|
||||
def _infer_schema_name(schema_version: str) -> str | None:
|
||||
marker = schema_version.rfind(".v")
|
||||
if marker <= 0:
|
||||
return None
|
||||
version = schema_version[marker + 2:]
|
||||
if not version or not version.isdigit():
|
||||
return None
|
||||
return schema_version[:marker]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import os
|
|||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import msgpack
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from pathlib import Path
|
||||
from cultcache_py.backing_store import CultCacheEnvelope
|
||||
|
|
@ -118,11 +119,6 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(cache.get_key_by_index(document, "category", "Consumable"), "item:first")
|
||||
|
||||
def test_database_entry_formatter_uses_slot_indexed_messagepack_array(self) -> None:
|
||||
try:
|
||||
import msgpack # type: ignore
|
||||
except ModuleNotFoundError:
|
||||
self.skipTest("msgpack optional dependency is not installed")
|
||||
|
||||
@dataclass
|
||||
class Settings:
|
||||
theme: str
|
||||
|
|
@ -157,11 +153,6 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual([member.slot for member in first.members], [0, 2])
|
||||
|
||||
def test_messagepack_store_writes_v1_snapshot(self) -> None:
|
||||
try:
|
||||
import msgpack # type: ignore
|
||||
except ModuleNotFoundError:
|
||||
self.skipTest("msgpack optional dependency is not installed")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
document = define_database_entry_type(
|
||||
"settings",
|
||||
|
|
@ -186,12 +177,7 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(raw[2][0][0], "app")
|
||||
self.assertEqual(raw[2][0][1], "settings")
|
||||
|
||||
def test_messagepack_store_recovers_schema_stamped_record_missing_catalog_entry(self) -> None:
|
||||
try:
|
||||
import msgpack # type: ignore
|
||||
except ModuleNotFoundError:
|
||||
self.skipTest("msgpack optional dependency is not installed")
|
||||
|
||||
def test_messagepack_store_refuses_a_schema_id_missing_from_the_catalog(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
document = define_database_entry_type(
|
||||
"runtime-policy",
|
||||
|
|
@ -227,11 +213,9 @@ class CultCacheTests(unittest.TestCase):
|
|||
.build()
|
||||
)
|
||||
|
||||
cache.pull_all_backing_stores()
|
||||
self.assertEqual(cache.get_required(document, "record-1")["value"], "still readable")
|
||||
envelope = cache.get_required_envelope(document, "record-1")
|
||||
self.assertEqual(envelope.type, "runtime-policy")
|
||||
self.assertEqual(envelope.schema_id, "sha256:stale-schema-id-from-cold-record")
|
||||
with self.assertRaisesRegex(ValueError, "references missing schema id"):
|
||||
cache.pull_all_backing_stores()
|
||||
self.assertEqual(cache.get_all(document), [])
|
||||
|
||||
def test_interop_cli_helpers_round_trip_v1_store(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import inspect
|
||||
import shutil
|
||||
import tempfile
|
||||
import traceback
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import msgpack
|
||||
|
||||
from cultcache_py import (
|
||||
JSON_PAYLOAD_CODEC,
|
||||
|
|
@ -13,14 +18,47 @@ from cultcache_py import (
|
|||
CultCacheEnvelope,
|
||||
CultCachePayloadDepthError,
|
||||
CultCachePayloadFormatError,
|
||||
DocumentDefinition,
|
||||
JsonLinesBackingStore,
|
||||
PayloadCodec,
|
||||
PayloadCodecFault,
|
||||
SingleFileMessagePackBackingStore,
|
||||
define_database_entry_type,
|
||||
define_document_type,
|
||||
migrate_payloads,
|
||||
)
|
||||
from cultcache_py.cache import CultCacheError
|
||||
|
||||
CANARY = "CANARY-payload-codec-5b81e0"
|
||||
LEGACY_JSON = b'{"n": 1, "schema": "x"}'
|
||||
KEY_KINDS = "str, int, float, bool or timestamp keys"
|
||||
|
||||
# The rows of the C# reference's object slot (map P2, jobK): the bytes of one map.
|
||||
ACCEPTED_KEY_ROWS = {
|
||||
"int": "810101",
|
||||
"negint": "81ff01",
|
||||
"u64max": "81cfffffffffffffffff01",
|
||||
"i64min": "81d3800000000000000001",
|
||||
"f32": "81ca3fc0000001",
|
||||
"f64": "81cb3ff800000000000001",
|
||||
"bool": "81c301",
|
||||
"tskey": "81d6ff0000000001",
|
||||
"str-and-int": "82a1610102 02".replace(" ", ""),
|
||||
}
|
||||
REFUSED_KIND_ROWS = {
|
||||
"nil": ("81c001", "nil"),
|
||||
"bin": ("81c4016b01", "bin"),
|
||||
"array": ("81910101", "array"),
|
||||
"map": ("8181010101", "map"),
|
||||
"ext": ("81d4010001", "ext"),
|
||||
}
|
||||
DUPLICATE_ROWS = {
|
||||
"dup-int": "82010101 02".replace(" ", ""),
|
||||
"dup-str": "82a16101a16102",
|
||||
"fix1-u8-1": "8201 01cc0102".replace(" ", ""),
|
||||
"int1-f64-1": "820101cb3ff000000000000002",
|
||||
"int1-true": "820101c302",
|
||||
}
|
||||
|
||||
|
||||
def _store_with(path: Path, payload: bytes) -> SingleFileMessagePackBackingStore:
|
||||
|
|
@ -29,6 +67,10 @@ def _store_with(path: Path, payload: bytes) -> SingleFileMessagePackBackingStore
|
|||
return store
|
||||
|
||||
|
||||
def _chain_text(error: BaseException) -> str:
|
||||
return "".join(traceback.format_exception(error)) + str(error) + repr(error)
|
||||
|
||||
|
||||
class PayloadCodecTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.directory = Path(tempfile.mkdtemp())
|
||||
|
|
@ -37,18 +79,30 @@ class PayloadCodecTests(unittest.TestCase):
|
|||
def _cache(self, document, store) -> CultCache:
|
||||
return CultCache.builder().register_document_type(document).add_generic_store(store).build()
|
||||
|
||||
def _refused(self, call, *args) -> CultCachePayloadFormatError:
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
call(*args)
|
||||
return caught.exception
|
||||
|
||||
def test_the_default_is_messagepack_for_both_definitions(self) -> None:
|
||||
document = define_document_type("tests.x")
|
||||
self.assertIs(document.payload_codec, MESSAGEPACK_PAYLOAD_CODEC)
|
||||
self.assertEqual(document.encode_payload({"a": [1]}), bytes.fromhex("81a1619101"))
|
||||
self.assertIs(define_database_entry_type("tests.entry", [("a", 0)]).payload_codec, MESSAGEPACK_PAYLOAD_CODEC)
|
||||
|
||||
def test_the_default_has_one_author(self) -> None:
|
||||
self.assertIs(DocumentDefinition("tests.x").payload_codec, MESSAGEPACK_PAYLOAD_CODEC)
|
||||
self.assertIs(define_document_type("tests.x").payload_codec, MESSAGEPACK_PAYLOAD_CODEC)
|
||||
self.assertIs(define_document_type("tests.x", payload_codec=JSON_PAYLOAD_CODEC).payload_codec, JSON_PAYLOAD_CODEC)
|
||||
self.assertIsNone(inspect.signature(define_document_type).parameters["payload_codec"].default)
|
||||
|
||||
def test_the_json_codec_writes_the_0_3_bytes(self) -> None:
|
||||
written = JSON_PAYLOAD_CODEC.encode({"b": "é", "a": [1, 2]})
|
||||
self.assertEqual(written, b'{"a": [1, 2], "b": "\xc3\xa9"}')
|
||||
self.assertEqual(JSON_PAYLOAD_CODEC.decode(written), {"a": [1, 2], "b": "é"})
|
||||
self.assertEqual(JSON_PAYLOAD_CODEC.encode({1: "a", "n": 2**70}), b'{"1": "a", "n": 1180591620717411303424}')
|
||||
|
||||
def test_a_json_store_is_refused_typed_with_the_remedy_and_loads_nothing(self) -> None:
|
||||
def test_a_json_store_is_refused_typed_and_loads_nothing(self) -> None:
|
||||
store = _store_with(self.directory / "legacy.cc", LEGACY_JSON)
|
||||
document = define_document_type("tests.legacy")
|
||||
cache = self._cache(document, store)
|
||||
|
|
@ -56,13 +110,53 @@ class PayloadCodecTests(unittest.TestCase):
|
|||
cache.pull_all_backing_stores()
|
||||
message = str(caught.exception)
|
||||
self.assertEqual((caught.exception.type_name, caught.exception.codec, caught.exception.direction), ("tests.legacy", "MessagePack", "read"))
|
||||
self.assertEqual(caught.exception.code, "not_one_value")
|
||||
self.assertIn("tests.legacy", message)
|
||||
self.assertIn("MessagePack", message)
|
||||
self.assertIn("payload_codec=JSON_PAYLOAD_CODEC", message)
|
||||
self.assertNotIn("schema", message)
|
||||
self.assertEqual(cache.get_all(document), [])
|
||||
|
||||
def test_a_json_store_opted_in_reads_and_migrates_to_messagepack(self) -> None:
|
||||
def test_the_refusal_texts_are_exact_with_their_codes(self) -> None:
|
||||
read = self._refused(define_document_type("tests.text").decode_payload, LEGACY_JSON)
|
||||
self.assertEqual(str(read), "tests.text payload is not one MessagePack value (ExtraData); refused on read.")
|
||||
self.assertEqual(read.code, "not_one_value")
|
||||
entry = self._refused(define_database_entry_type("tests.entry", [("a", 0)]).decode_payload, LEGACY_JSON)
|
||||
self.assertEqual(str(entry), "tests.entry payload is not one MessagePack value (ExtraData); refused on read.")
|
||||
json_side = self._refused(define_document_type("tests.text", payload_codec=JSON_PAYLOAD_CODEC).decode_payload, b"\x81\xa1a")
|
||||
self.assertEqual(str(json_side), "tests.text payload is not one JSON value (UnicodeDecodeError); refused on read.")
|
||||
self.assertEqual((json_side.type_name, json_side.codec, json_side.direction), ("tests.text", "JSON", "read"))
|
||||
nil = self._refused(define_document_type("tests.text").decode_payload, bytes.fromhex("81c001"))
|
||||
self.assertEqual(
|
||||
str(nil),
|
||||
"tests.text payload has a map key of kind nil; MessagePack payloads take str, int, float, bool or timestamp keys (refused on read).",
|
||||
)
|
||||
self.assertEqual((nil.code, nil.direction), ("map_key_kind", "read"))
|
||||
duplicate = self._refused(define_document_type("tests.text").decode_payload, bytes.fromhex("82010101 02".replace(" ", "")))
|
||||
self.assertEqual(str(duplicate), "tests.text payload has two map keys Python holds as one; refused on read.")
|
||||
self.assertEqual(duplicate.code, "duplicate_map_key")
|
||||
key = self._refused(define_document_type("tests.text", encode=lambda value: value).encode_payload, {b"k": 1})
|
||||
self.assertEqual(
|
||||
str(key),
|
||||
"tests.text payload has a map key of kind bin; MessagePack payloads take str, int, float, bool or timestamp keys (refused on write).",
|
||||
)
|
||||
self.assertEqual((key.type_name, key.codec, key.direction, key.code), ("tests.text", "MessagePack", "write", "map_key_kind"))
|
||||
big = self._refused(define_document_type("tests.text", encode=lambda value: value).encode_payload, {"a": 2**64})
|
||||
self.assertEqual(str(big), "tests.text payload has an integer beyond 64 bits; refused on write.")
|
||||
self.assertEqual((big.code, big.direction), ("integer_out_of_range", "write"))
|
||||
|
||||
def test_a_consumer_codec_fault_with_its_own_code_is_a_typed_refusal_naming_it(self) -> None:
|
||||
def encode(tree: object) -> bytes:
|
||||
raise PayloadCodecFault("no_sets", "set")
|
||||
|
||||
document = define_document_type("tests.own", payload_codec=PayloadCodec("Own", encode, lambda payload: None))
|
||||
refusal = self._refused(document.encode_payload, {"a": 1})
|
||||
self.assertEqual(str(refusal), "tests.own payload is refused by the Own codec (no_sets); refused on write.")
|
||||
self.assertEqual((refusal.codec, refusal.code, refusal.direction), ("Own", "no_sets", "write"))
|
||||
fault = PayloadCodecFault("no_sets", "set")
|
||||
self.assertEqual((fault.code, fault.kind), ("no_sets", "set"))
|
||||
self.assertIsInstance(fault, ValueError)
|
||||
|
||||
def test_a_json_store_opted_in_reads_and_migrate_payloads_rewrites_it(self) -> None:
|
||||
path = self.directory / "legacy.cc"
|
||||
store = _store_with(path, LEGACY_JSON)
|
||||
legacy = define_document_type("tests.legacy", payload_codec=JSON_PAYLOAD_CODEC)
|
||||
|
|
@ -70,31 +164,12 @@ class PayloadCodecTests(unittest.TestCase):
|
|||
old.pull_all_backing_stores()
|
||||
self.assertEqual(old.get(legacy, "k"), {"n": 1, "schema": "x"})
|
||||
modern = define_document_type("tests.legacy")
|
||||
new = self._cache(modern, SingleFileMessagePackBackingStore(path))
|
||||
new.put(modern, "k", old.get(legacy, "k"))
|
||||
self.assertEqual(migrate_payloads(SingleFileMessagePackBackingStore(path), [modern]), 1)
|
||||
reread = self._cache(modern, SingleFileMessagePackBackingStore(path))
|
||||
reread.pull_all_backing_stores()
|
||||
self.assertEqual(reread.get(modern, "k"), {"n": 1, "schema": "x"})
|
||||
self.assertEqual(reread.get_envelope(modern, "k").payload, MESSAGEPACK_PAYLOAD_CODEC.encode({"n": 1, "schema": "x"}))
|
||||
|
||||
def test_the_read_refusal_text_is_exact_under_each_codec(self) -> None:
|
||||
with self.assertRaises(CultCachePayloadFormatError) as default:
|
||||
define_document_type("tests.text").decode_payload(LEGACY_JSON)
|
||||
self.assertEqual(
|
||||
str(default.exception),
|
||||
"tests.text payload is not one MessagePack value (ExtraData); refused on read. A record written by cultcache-py 0.3 "
|
||||
"or earlier through define_document_type's JSON default is read by defining tests.text with payload_codec=JSON_PAYLOAD_CODEC.",
|
||||
)
|
||||
with self.assertRaises(CultCachePayloadFormatError) as json_side:
|
||||
define_document_type("tests.text", payload_codec=JSON_PAYLOAD_CODEC).decode_payload(b"\x81\xa1a")
|
||||
self.assertEqual(str(json_side.exception), "tests.text payload is not one JSON value (UnicodeDecodeError); refused on read.")
|
||||
self.assertEqual((json_side.exception.type_name, json_side.exception.codec, json_side.exception.direction), ("tests.text", "JSON", "read"))
|
||||
|
||||
def test_the_write_refusal_carries_its_type_codec_and_direction(self) -> None:
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
define_document_type("tests.attrs", encode=lambda value: value).encode_payload({1: "a"})
|
||||
self.assertEqual((caught.exception.type_name, caught.exception.codec, caught.exception.direction), ("tests.attrs", "MessagePack", "write"))
|
||||
|
||||
def test_wrong_format_payloads_are_typed_refusals_on_both_sides(self) -> None:
|
||||
document = define_document_type("tests.faults")
|
||||
for payload in (b"[1,2]", b'"hi"', b"true", b"null", b"1.5", b"-3", b"12", b"\x93\x01\x02", b"\xc1", b"\xa2\xff\xfe"):
|
||||
|
|
@ -102,21 +177,9 @@ class PayloadCodecTests(unittest.TestCase):
|
|||
with self.assertRaises(CultCachePayloadFormatError):
|
||||
document.decode_payload(payload)
|
||||
json_document = define_document_type("tests.faults", payload_codec=JSON_PAYLOAD_CODEC)
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
json_document.decode_payload(MESSAGEPACK_PAYLOAD_CODEC.encode({"a": 1}))
|
||||
self.assertNotIn("JSON_PAYLOAD_CODEC", str(caught.exception))
|
||||
self.assertIn("JSON", str(caught.exception))
|
||||
|
||||
def test_the_refusals_never_echo_the_payload_or_the_key(self) -> None:
|
||||
document = define_document_type("tests.canary")
|
||||
with self.assertRaises(CultCachePayloadFormatError) as read:
|
||||
document.decode_payload(('{"' + CANARY + '": 1}').encode())
|
||||
self.assertNotIn(CANARY, str(read.exception))
|
||||
with self.assertRaises(CultCachePayloadFormatError) as write:
|
||||
define_document_type("tests.canary", encode=lambda value: value).encode_payload({7351904: CANARY})
|
||||
self.assertNotIn("7351904", str(write.exception))
|
||||
self.assertNotIn(CANARY, str(write.exception))
|
||||
self.assertIsNone(read.exception.__cause__)
|
||||
caught = self._refused(json_document.decode_payload, MESSAGEPACK_PAYLOAD_CODEC.encode({"a": 1}))
|
||||
self.assertNotIn("JSON_PAYLOAD_CODEC", str(caught))
|
||||
self.assertIn("JSON", str(caught))
|
||||
|
||||
def test_a_one_digit_json_text_reads_as_a_messagepack_integer(self) -> None:
|
||||
self.assertEqual(define_document_type("tests.digit").decode_payload(b"5"), 53)
|
||||
|
|
@ -138,34 +201,17 @@ class PayloadCodecTests(unittest.TestCase):
|
|||
raise KeyError(CANARY)
|
||||
|
||||
document = define_document_type("tests.v", payload_codec=PayloadCodec("t", lambda tree: b"", lambda payload: int("x")))
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
document.decode_payload(b"x")
|
||||
self.assertIn("(ValueError)", str(caught.exception))
|
||||
self.assertIn("(ValueError)", str(self._refused(document.decode_payload, b"x")))
|
||||
passthrough = define_document_type("tests.v", payload_codec=PayloadCodec("t", lambda tree: b"", refuse))
|
||||
with self.assertRaises(KeyError):
|
||||
passthrough.decode_payload(b"x")
|
||||
|
||||
def test_map_keys_the_default_cannot_read_back_are_refused_on_write(self) -> None:
|
||||
document = define_document_type("tests.keys", encode=lambda value: value)
|
||||
for tree, key_type in (({1: "a"}, "int"), ({True: 1}, "bool"), ({"a": {2: 3}}, "int"), ({"a": [{1.5: 1}]}, "float")):
|
||||
with self.subTest(tree=tree):
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
document.encode_payload(tree)
|
||||
self.assertEqual(
|
||||
str(caught.exception),
|
||||
f"tests.keys payload has a map key of type {key_type}; MessagePack payloads take str or bytes keys (refused on write).",
|
||||
)
|
||||
tree = {"a": 1, b"b": {"c": [2]}}
|
||||
self.assertEqual(document.decode_payload(document.encode_payload(tree)), tree)
|
||||
legacy = define_document_type("tests.keys", encode=lambda value: value, payload_codec=JSON_PAYLOAD_CODEC)
|
||||
self.assertEqual(legacy.encode_payload({1: "a"}), b'{"1": "a"}')
|
||||
|
||||
def test_a_refused_key_reaches_no_store(self) -> None:
|
||||
document = define_document_type("tests.keys", encode=lambda value: value)
|
||||
store = SingleFileMessagePackBackingStore(self.directory / "keys.cc")
|
||||
cache = self._cache(document, store)
|
||||
with self.assertRaises(CultCachePayloadFormatError):
|
||||
cache.put(document, "k", {1: "a"})
|
||||
cache.put(document, "k", {b"k": "a"})
|
||||
self.assertEqual(store.pull_all(), [])
|
||||
|
||||
def test_the_library_codecs_refuse_nodes_the_depth_walk_cannot_see(self) -> None:
|
||||
|
|
@ -181,6 +227,265 @@ class PayloadCodecTests(unittest.TestCase):
|
|||
with self.assertRaises(TypeError):
|
||||
document.encode_payload({"a": node})
|
||||
|
||||
# The key rule is the C# reference's (map P2), both ways.
|
||||
|
||||
def test_reference_map_keys_read_through_an_any_slot(self) -> None:
|
||||
entry = define_database_entry_type("tests.any", [("v", 0)])
|
||||
for name, row in ACCEPTED_KEY_ROWS.items():
|
||||
with self.subTest(row=name):
|
||||
value = entry.decode_payload(b"\x91" + bytes.fromhex(row))["v"]
|
||||
again = entry.decode_payload(entry.encode_payload({"v": value}))["v"]
|
||||
self.assertEqual(again, value)
|
||||
self.assertEqual([type(key) for key in again], [type(key) for key in value])
|
||||
sixty_four = entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["u64max"]))["v"]
|
||||
self.assertEqual(list(sixty_four), [2**64 - 1])
|
||||
self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["i64min"]))["v"]), [-(2**63)])
|
||||
self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["bool"]))["v"]), [True])
|
||||
self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["f64"]))["v"]), [1.5])
|
||||
self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["tskey"]))["v"]), [msgpack.Timestamp(0, 0)])
|
||||
mixed = entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["str-and-int"]))["v"]
|
||||
self.assertEqual(mixed, {"a": 1, 2: 2})
|
||||
for name, (row, kind) in REFUSED_KIND_ROWS.items():
|
||||
with self.subTest(row=name):
|
||||
refusal = self._refused(entry.decode_payload, b"\x91" + bytes.fromhex(row))
|
||||
self.assertEqual((refusal.code, refusal.direction), ("map_key_kind", "read"))
|
||||
self.assertIn(f"map key of kind {kind};", str(refusal))
|
||||
for name, row in DUPLICATE_ROWS.items():
|
||||
with self.subTest(row=name):
|
||||
refusal = self._refused(entry.decode_payload, b"\x91" + bytes.fromhex(row))
|
||||
self.assertEqual(refusal.code, "duplicate_map_key")
|
||||
|
||||
def test_reference_map_keys_write_and_read_back_through_a_document_type(self) -> None:
|
||||
document = define_document_type("tests.keys", encode=lambda value: value)
|
||||
for tree in ({1: "a"}, {True: 1}, {1.5: 1}, {msgpack.Timestamp(0, 0): 1}, {"a": {2: 3}}, {"a": [{1.5: 1}]}, {-(2**63): 1}, {2**64 - 1: 1}):
|
||||
with self.subTest(tree=tree):
|
||||
self.assertEqual(document.decode_payload(document.encode_payload(tree)), tree)
|
||||
refused = (
|
||||
({b"k": 1}, "bin"),
|
||||
({bytearray(b"k"): 1}, "bin"),
|
||||
({None: 1}, "nil"),
|
||||
({(1,): 1}, "array"),
|
||||
({frozenset(): 1}, "array"),
|
||||
({msgpack.ExtType(1, b"x"): 1}, "ext"),
|
||||
({object(): 1}, "unsupported"),
|
||||
({"a": [{b"k": 1}]}, "bin"),
|
||||
)
|
||||
for tree, kind in refused:
|
||||
with self.subTest(tree=tree):
|
||||
refusal = self._refused(document.encode_payload, tree)
|
||||
self.assertEqual((refusal.code, refusal.direction), ("map_key_kind", "write"))
|
||||
self.assertIn(f"map key of kind {kind};", str(refusal))
|
||||
legacy = define_document_type("tests.keys", encode=lambda value: value, payload_codec=JSON_PAYLOAD_CODEC)
|
||||
self.assertEqual(legacy.encode_payload({1: "a"}), b'{"1": "a"}')
|
||||
|
||||
def test_a_float32_key_writes_as_a_float_the_reference_reads(self) -> None:
|
||||
from cultcache_py.leaves import Float32
|
||||
|
||||
written = MESSAGEPACK_PAYLOAD_CODEC.encode({Float32(1.5): 1})
|
||||
self.assertEqual(written, bytes.fromhex("81ca3fc0000001"))
|
||||
self.assertEqual(MESSAGEPACK_PAYLOAD_CODEC.decode(written), {1.5: 1})
|
||||
|
||||
def test_the_codec_owns_its_keys_both_ways(self) -> None:
|
||||
codec = MESSAGEPACK_PAYLOAD_CODEC
|
||||
for tree, code, kind in (
|
||||
({b"k": 1}, "map_key_kind", "bin"),
|
||||
({None: 1}, "map_key_kind", "nil"),
|
||||
({"a": 2**64}, "integer_out_of_range", ""),
|
||||
):
|
||||
with self.subTest(tree=tree):
|
||||
with self.assertRaises(PayloadCodecFault) as caught:
|
||||
codec.encode(tree)
|
||||
self.assertEqual((caught.exception.code, caught.exception.kind), (code, kind))
|
||||
for name, (row, kind) in REFUSED_KIND_ROWS.items():
|
||||
with self.subTest(row=name):
|
||||
with self.assertRaises(PayloadCodecFault) as caught:
|
||||
codec.decode(bytes.fromhex(row))
|
||||
self.assertEqual((caught.exception.code, caught.exception.kind), ("map_key_kind", kind))
|
||||
for name, row in DUPLICATE_ROWS.items():
|
||||
with self.subTest(row=name):
|
||||
with self.assertRaises(PayloadCodecFault) as caught:
|
||||
codec.decode(bytes.fromhex(row))
|
||||
self.assertEqual((caught.exception.code, caught.exception.kind), ("duplicate_map_key", ""))
|
||||
for tree in ({1: "a"}, {True: 1}, {1.5: [2**64 - 1, -(2**63)]}, {"a": {-1: None}}, [{2: 3}]):
|
||||
with self.subTest(tree=tree):
|
||||
self.assertEqual(codec.decode(codec.encode(tree)), tree)
|
||||
|
||||
def test_a_cyclic_tree_is_not_walked_forever_by_the_codec(self) -> None:
|
||||
cycle: dict = {}
|
||||
cycle["a"] = cycle
|
||||
with self.assertRaises(RecursionError):
|
||||
MESSAGEPACK_PAYLOAD_CODEC.encode(cycle)
|
||||
with self.assertRaises(CultCachePayloadDepthError):
|
||||
define_document_type("tests.cycle", encode=lambda value: value).encode_payload(cycle)
|
||||
|
||||
# Integers beyond 64 bits are a format limit with a typed refusal.
|
||||
|
||||
def test_integers_beyond_64_bits_are_refused_typed_on_write(self) -> None:
|
||||
document = define_document_type("tests.big", encode=lambda value: value)
|
||||
store = SingleFileMessagePackBackingStore(self.directory / "big.cc")
|
||||
cache = self._cache(document, store)
|
||||
for value in (2**64, -(2**63) - 1, 2**64 + 987654):
|
||||
for name, tree in (("value", {"a": value}), ("list", {"a": [[value]]}), ("key", {value: 1}), ("root", value)):
|
||||
with self.subTest(value=value, place=name):
|
||||
refusal = self._refused(document.encode_payload, tree)
|
||||
self.assertEqual((refusal.code, refusal.direction), ("integer_out_of_range", "write"))
|
||||
text = _chain_text(refusal)
|
||||
self.assertNotIn(str(value), text)
|
||||
self.assertNotIn(str(value)[:10], text)
|
||||
with self.assertRaises(CultCachePayloadFormatError):
|
||||
cache.put(document, "k", {"a": value})
|
||||
self.assertEqual(store.pull_all(), [])
|
||||
for value in (2**64 - 1, -(2**63)):
|
||||
for tree in ({"a": value}, {"a": [value]}, {value: 1}):
|
||||
with self.subTest(value=value, tree=tree):
|
||||
self.assertEqual(document.decode_payload(document.encode_payload(tree)), tree)
|
||||
|
||||
# A refusal carries no payload byte or value, on its text or on its exception chain.
|
||||
|
||||
def test_refusals_carry_no_payload_on_their_text_or_exception_chain(self) -> None:
|
||||
raw = CANARY.encode()
|
||||
text_key = bytes([0xA0 + len(raw)]) + raw
|
||||
read_rows = {
|
||||
"extra-data": (define_document_type("tests.c"), b"\x01" + raw),
|
||||
"utf8": (define_document_type("tests.c", payload_codec=JSON_PAYLOAD_CODEC), b"\xff" + raw),
|
||||
"json-text": (define_document_type("tests.c", payload_codec=JSON_PAYLOAD_CODEC), b'{"' + raw + b'": '),
|
||||
"map-key-kind": (define_document_type("tests.c"), b"\x81\xc4" + bytes([len(raw)]) + raw + b"\x01"),
|
||||
"duplicate-key": (define_document_type("tests.c"), b"\x82" + text_key + b"\x01" + text_key + b"\x02"),
|
||||
"depth": (define_document_type("tests.c"), b"\x91" * 300 + text_key),
|
||||
"not-msgpack": (define_document_type("tests.c"), b"\xc1" + raw),
|
||||
}
|
||||
for name, (document, payload) in read_rows.items():
|
||||
with self.subTest(row=name):
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
document.decode_payload(payload)
|
||||
self.assertIsNone(caught.exception.__context__)
|
||||
self.assertIsNone(caught.exception.__cause__)
|
||||
self.assertNotIn(CANARY, _chain_text(caught.exception))
|
||||
write = define_document_type("tests.c", encode=lambda value: value)
|
||||
for name, tree in (
|
||||
("map-key-kind", {raw: 1}),
|
||||
("integer", {"k": 2**64 + 987654, CANARY: 1}),
|
||||
("nested", {"a": [{raw: CANARY}]}),
|
||||
):
|
||||
with self.subTest(row=name):
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
write.encode_payload(tree)
|
||||
self.assertIsNone(caught.exception.__context__)
|
||||
self.assertIsNone(caught.exception.__cause__)
|
||||
self.assertNotIn(CANARY, _chain_text(caught.exception))
|
||||
with self.assertRaises(CultCachePayloadDepthError) as deep:
|
||||
define_document_type("tests.c", encode=lambda value: value).encode_payload(self._nested(300, CANARY))
|
||||
self.assertIsNone(deep.exception.__context__)
|
||||
self.assertNotIn(CANARY, _chain_text(deep.exception))
|
||||
|
||||
@staticmethod
|
||||
def _nested(depth: int, leaf: object) -> object:
|
||||
tree = leaf
|
||||
for _ in range(depth):
|
||||
tree = [tree]
|
||||
return tree
|
||||
|
||||
# The legacy envelope reader extracts bytes; the registered codec still decides.
|
||||
|
||||
def test_a_legacy_inline_payload_is_messagepack_and_a_json_definition_refuses_it(self) -> None:
|
||||
path = self.directory / "inline.cc"
|
||||
path.write_bytes(
|
||||
msgpack.packb([{"key": "k", "type": "tests.legacy", "payload": {"n": 1}, "storedAt": "2026-01-01T00:00:00Z"}], use_bin_type=True)
|
||||
)
|
||||
document = define_document_type("tests.legacy")
|
||||
cache = self._cache(document, SingleFileMessagePackBackingStore(path))
|
||||
cache.pull_all_backing_stores()
|
||||
self.assertEqual(cache.get(document, "k"), {"n": 1})
|
||||
json_document = define_document_type("tests.legacy", payload_codec=JSON_PAYLOAD_CODEC)
|
||||
refused = self._cache(json_document, SingleFileMessagePackBackingStore(path))
|
||||
with self.assertRaises(CultCachePayloadFormatError) as caught:
|
||||
refused.pull_all_backing_stores()
|
||||
self.assertEqual(caught.exception.code, "not_one_value")
|
||||
|
||||
|
||||
MIGRATION_RECORDS = {
|
||||
"k1": b'{"n": 1}',
|
||||
"k2": b'["a", 2]',
|
||||
"k3": b'{"s": "\xc3\xa9"}',
|
||||
"k4": b'{"a": [1, 2], "b": {"c": true}}',
|
||||
"k5": b'{"t": null, "f": 1.5}',
|
||||
"k6": b'{"x": {"y": [[0, 1], []]}}',
|
||||
"k7": b'{"big": 1180591620717411303424}',
|
||||
}
|
||||
BIG_DIGITS = "1180591620717411303424"
|
||||
|
||||
|
||||
class MigratePayloadsTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.directory = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, self.directory, True)
|
||||
|
||||
def _stores(self):
|
||||
yield "single-file", lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc")
|
||||
yield "jsonl", lambda name: JsonLinesBackingStore(self.directory / f"{name}.jsonl")
|
||||
|
||||
def _filled(self, make, name: str, records: dict[str, bytes]):
|
||||
store = make(name)
|
||||
for key, payload in records.items():
|
||||
store.push(CultCacheEnvelope.create(key=key, type="tests.mig", payload=payload))
|
||||
return store
|
||||
|
||||
def _pull_with(self, store, codec):
|
||||
document = define_document_type("tests.mig", payload_codec=codec)
|
||||
cache = CultCache.builder().register_document_type(document).add_generic_store(store).build()
|
||||
cache.pull_all_backing_stores()
|
||||
return cache.get_all(document)
|
||||
|
||||
def test_migration_is_refused_whole_and_leaves_the_store_untouched(self) -> None:
|
||||
for label, make in self._stores():
|
||||
with self.subTest(store=label):
|
||||
store = self._filled(make, f"refused-{label}", MIGRATION_RECORDS)
|
||||
before = store.path.read_bytes()
|
||||
with self.assertRaises(CultCacheError) as caught:
|
||||
migrate_payloads(store, [define_document_type("tests.mig")])
|
||||
message = str(caught.exception)
|
||||
for part in ("tests.mig", "k7", "integer_out_of_range"):
|
||||
self.assertIn(part, message)
|
||||
self.assertNotIn(BIG_DIGITS[:8], _chain_text(caught.exception))
|
||||
self.assertIsNone(caught.exception.__context__)
|
||||
self.assertEqual(store.path.read_bytes(), before)
|
||||
self.assertEqual(len(self._pull_with(store, JSON_PAYLOAD_CODEC)), 7)
|
||||
|
||||
def test_migration_rewrites_every_listed_record_in_one_write(self) -> None:
|
||||
records = {key: payload for key, payload in MIGRATION_RECORDS.items() if key != "k7"}
|
||||
for label, make in self._stores():
|
||||
with self.subTest(store=label):
|
||||
store = self._filled(make, f"ok-{label}", records)
|
||||
other = CultCacheEnvelope.create(key="o", type="tests.other", payload=b"opaque\x00bytes")
|
||||
store.push(other)
|
||||
with mock.patch.object(store, "push_all", wraps=store.push_all) as push_all, mock.patch.object(
|
||||
store, "push", wraps=store.push
|
||||
) as push:
|
||||
count = migrate_payloads(store, [define_document_type("tests.mig")])
|
||||
self.assertEqual(count, 6)
|
||||
self.assertEqual(push_all.call_count, 1)
|
||||
self.assertEqual(push.call_count, 0)
|
||||
kept = next(item for item in store.pull_all() if item.type == "tests.other")
|
||||
self.assertEqual((kept.payload, kept.stored_at), (other.payload, other.stored_at))
|
||||
default = self._pull_with(store, MESSAGEPACK_PAYLOAD_CODEC)
|
||||
self.assertEqual(len(default), 6)
|
||||
with self.assertRaises(CultCachePayloadFormatError):
|
||||
self._pull_with(store, JSON_PAYLOAD_CODEC)
|
||||
migrated = {item.key: item.payload for item in store.pull_all() if item.type == "tests.mig"}
|
||||
self.assertEqual(migrated["k1"], MESSAGEPACK_PAYLOAD_CODEC.encode({"n": 1}))
|
||||
self.assertEqual(migrated["k2"], MESSAGEPACK_PAYLOAD_CODEC.encode(["a", 2]))
|
||||
after = store.path.read_bytes()
|
||||
with self.assertRaises(CultCacheError):
|
||||
migrate_payloads(store, [define_document_type("tests.mig")])
|
||||
self.assertEqual(store.path.read_bytes(), after)
|
||||
|
||||
def test_migration_names_a_source_codec_and_skips_unlisted_types(self) -> None:
|
||||
store = self._filled(lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc"), "src", {"k1": b'{"n": 1}'})
|
||||
self.assertEqual(migrate_payloads(store, [define_document_type("tests.nothing")]), 0)
|
||||
self.assertEqual(store.pull_all()[0].payload, b'{"n": 1}')
|
||||
self.assertEqual(migrate_payloads(store, [define_document_type("tests.mig")], source_codec=JSON_PAYLOAD_CODEC), 1)
|
||||
self.assertEqual(store.pull_all()[0].payload, MESSAGEPACK_PAYLOAD_CODEC.encode({"n": 1}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
import msgpack
|
||||
from cultcache_py import CultCache, CultCacheEnvelope
|
||||
from cultcache_py.documents import DocumentDefinition
|
||||
|
||||
|
|
@ -184,8 +185,10 @@ def resolve_document_and_schema_id_for_raw_record(
|
|||
|
||||
|
||||
def _infer_schema_version_from_payload(payload: bytes) -> str | None:
|
||||
"""Routing only, as the C# `CultNetDocumentRegistry.TryResolveDescriptorByPayloadSchema` does: reads a schema stamp
|
||||
off a raw record whose schema id is unknown, to pick a definition. It decides no format; the definition it picks
|
||||
has a codec that decodes the payload or refuses it."""
|
||||
try:
|
||||
import msgpack # type: ignore
|
||||
decoded = msgpack.unpackb(payload, raw=False)
|
||||
except Exception:
|
||||
return None
|
||||
|
|
|
|||
Loading…
Reference in a new issue