diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index f9f70b1f..4050cf16 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import struct from dataclasses import MISSING, dataclass, field from collections.abc import Mapping, Sequence, Set from itertools import chain @@ -44,7 +45,7 @@ class CultCachePayloadDepthError(ValueError): class PayloadCodecFault(ValueError): """A codec raises this for a payload its format cannot carry. `code` names the rule (the library's are - `map_key_kind`, `duplicate_map_key` and `integer_out_of_range`) and `kind` the kind of node at fault (a MessagePack + `map_key_kind`, `duplicate_map_key`, `integer_out_of_range` and `depth_limit`) and `kind` the kind of node at fault (a MessagePack kind such as `nil` or `bin`); neither is ever a payload value. A consumer-built codec may raise it.""" def __init__(self, code: str, kind: str = "") -> None: @@ -71,7 +72,7 @@ class CultCachePayloadFormatError(ValueError): f"{codec} payloads take str, int, float, bool or timestamp keys (refused on {direction})." ) elif code == "duplicate_map_key": - message = f"{type_name} payload has two map keys Python holds as one; refused on read." + message = f"{type_name} payload has two map keys Python holds as one; refused on {direction}." elif code == "integer_out_of_range": message = f"{type_name} payload has an integer beyond 64 bits; refused on write." else: @@ -79,20 +80,38 @@ class CultCachePayloadFormatError(ValueError): super().__init__(message) -def _check_payload_tree(tree: Any, type_name: str, direction: str) -> None: - """The one depth walk over a payload tree, for every codec. Iterative (every mapping, sequence and set that is not - text or bytes is a container): refuses the first container past `MAX_PAYLOAD_DEPTH`.""" +def _is_container(node: Any) -> bool: + return isinstance(node, _CONTAINERS) and not isinstance(node, _SCALAR_SEQUENCES) + + +def _children(node: Any) -> Iterable[Any]: + return chain(node.keys(), node.values()) if isinstance(node, Mapping) else node + + +def _containers(tree: Any) -> Iterable[Any]: + """The one depth walk over a payload tree, for every codec and every caller. Iterative, so a cycle ends at the limit: + yields every container (a mapping, sequence or set that is not text or bytes; a mapping's keys and values are both + children), the outermost at depth 1, and raises `PayloadCodecFault('depth_limit')` at the first one past + `MAX_PAYLOAD_DEPTH`.""" stack = [(tree, 1)] if _is_container(tree) else [] while stack: node, depth = stack.pop() if depth > MAX_PAYLOAD_DEPTH: - raise CultCachePayloadDepthError(type_name, direction) - children = chain(node.keys(), node.values()) if isinstance(node, Mapping) else node - stack.extend((child, depth + 1) for child in children if _is_container(child)) + raise PayloadCodecFault("depth_limit") + yield node + stack.extend((child, depth + 1) for child in _children(node) if _is_container(child)) -def _is_container(node: Any) -> bool: - return isinstance(node, _CONTAINERS) and not isinstance(node, _SCALAR_SEQUENCES) +def _check_payload_tree(tree: Any, type_name: str, direction: str) -> None: + """The definition's translation of the depth walk into its typed refusal.""" + try: + for _ in _containers(tree): + pass + except PayloadCodecFault: + refusal = CultCachePayloadDepthError(type_name, direction) + else: + return + raise refusal def _default_encode(value: Any) -> Any: @@ -139,8 +158,8 @@ def _pack_tree(tree: Any) -> bytes: @dataclass(frozen=True) class PayloadCodec: """A codec owns one payload format: how a tree becomes bytes and back, which of its decode faults mean the payload - nests past the interpreter's own stack, and which trees it can carry: it raises `PayloadCodecFault` for a key or an - integer its format cannot hold, and nothing it writes is refused by its own `decode`. A consumer-built codec owns + nests past the interpreter's own stack, and which trees it can carry: it raises `PayloadCodecFault` for a key, an + integer or a nesting its format cannot hold, and nothing it writes is refused by its own `decode`. A consumer-built codec owns the depth of nodes the depth walk cannot see (anything that is not a Mapping, Sequence or Set).""" name: str @@ -150,7 +169,6 @@ class PayloadCodec: _INTEGER_RANGE = (-(2**63), 2**64 - 1) -_PACK_DEPTH_LIMIT = 511 # msgpack's own packb limit: a tree nested past it, or a cycle, is not walked on _KEY_TYPES = (str, int, float, Float32, msgpack.Timestamp) @@ -181,24 +199,27 @@ def _checked_map(pairs: list[tuple[Any, Any]]) -> dict[Any, Any]: return held +def _key_image(key: Any) -> Any: + """A map key as the reader holds it: a `Float32` is written as float32 and read back as that float.""" + return struct.unpack(">f", struct.pack(">f", key.value))[0] if isinstance(key, Float32) else key + + def _check_writable(tree: Any) -> None: - """The write side of the key and integer rule: what `_checked_map` refuses on read is refused here, so nothing - this codec writes is refused by its own decode. Iterative; the nesting limit is the definition's rule (a tree past it - is the definition's refusal), so the walk stops only where `packb` would, which also ends a cycle.""" - stack = [(tree, 1)] - while stack: - node, depth = stack.pop() - if depth > _PACK_DEPTH_LIMIT: - raise ValueError("recursion limit exceeded") - if isinstance(node, int) and not _INTEGER_RANGE[0] <= node <= _INTEGER_RANGE[1]: - raise PayloadCodecFault("integer_out_of_range") + """The write side of the key, integer and depth rule: what `_checked_map` refuses on read, and nesting past + `MAX_PAYLOAD_DEPTH`, is refused here, so nothing this codec writes is refused by its own decode. Keys that collide + once decoded (a `Float32` beside an equal int, bool or float) are the reader's duplicate.""" + if isinstance(tree, int) and not _INTEGER_RANGE[0] <= tree <= _INTEGER_RANGE[1]: + raise PayloadCodecFault("integer_out_of_range") + for node in _containers(tree): + for child in _children(node): + if isinstance(child, int) and not _INTEGER_RANGE[0] <= child <= _INTEGER_RANGE[1]: + raise PayloadCodecFault("integer_out_of_range") if isinstance(node, Mapping): for key in node: if not isinstance(key, _KEY_TYPES): raise PayloadCodecFault("map_key_kind", _node_kind(key)) - stack.extend((child, depth + 1) for child in chain(node.keys(), node.values())) - elif isinstance(node, (list, tuple)): - stack.extend((child, depth + 1) for child in node) + if len(dict.fromkeys(_key_image(key) for key in node)) != len(node): + raise PayloadCodecFault("duplicate_map_key") def _unpack_tree(payload: bytes) -> Any: @@ -222,8 +243,8 @@ MESSAGEPACK_PAYLOAD_CODEC = PayloadCodec( "MessagePack", _pack_checked_tree, _unpack_tree, depth_faults=(msgpack.exceptions.StackError,) ) """The default payload format. It carries what the C# reference's object reader accepts: str, int, float, bool and -timestamp keys, no duplicate key, and integers within -(2**63)..2**64-1. It refuses the rest on read through its map -hook and on write through its encode, so it never writes a record it cannot read.""" +timestamp keys, no duplicate key, and integers within -(2**63)..2**64-1, nested no deeper than `MAX_PAYLOAD_DEPTH`. It refuses the rest on read through its +map hook and on write through its encode, so it never writes a record it cannot read.""" JSON_PAYLOAD_CODEC = PayloadCodec("JSON", _dump_json, _load_json, depth_faults=(RecursionError,)) """The 0.3 default, kept for stores written before MessagePack: it writes exactly the 0.3 bytes. Never a default.""" @@ -263,7 +284,9 @@ class UnionArm: value: Any = None -def _fault_refusal(type_name: str, codec: PayloadCodec, direction: str, fault: PayloadCodecFault) -> CultCachePayloadFormatError: +def _fault_refusal(type_name: str, codec: PayloadCodec, direction: str, fault: PayloadCodecFault) -> CultCachePayloadFormatError | CultCachePayloadDepthError: + if fault.code == "depth_limit": + return CultCachePayloadDepthError(type_name, direction) return CultCachePayloadFormatError(type_name, codec.name, direction, fault.code, fault.kind) diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 1d90b370..3c34c19f 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -14,6 +14,7 @@ from .backing_store import ( CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember, ) +from .documents import _unpack_tree STORE_FORMAT_VERSION = "cultcache.store.v1" _STORE_FORMAT_PREFIX = "cultcache.store." @@ -95,7 +96,7 @@ class SingleFileMessagePackBackingStore: data = _read_store(self.path, Path.read_bytes) if data is None: return [] - decoded = msgpack.unpackb(data, raw=False) + decoded = _unpack_tree(data) snapshot = _decode_v1_snapshot(decoded) if snapshot is not None: return snapshot diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index 7193d0c5..3ee3db89 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -2,6 +2,7 @@ import collections import dataclasses +import json import shutil import tempfile import traceback @@ -16,6 +17,8 @@ from cultcache_py import ( MAX_SLOT_KEY, CultCache, JSON_PAYLOAD_CODEC, + MESSAGEPACK_PAYLOAD_CODEC, + PayloadCodecFault, CultCachePayloadDepthError, DatabaseEntryField, PayloadCodec, @@ -645,8 +648,13 @@ class PayloadDepthRuleTests(unittest.TestCase): past = _nest(257, mixed) with self.assertRaises(CultCachePayloadDepthError): document.encode_payload(past) + raw = json.dumps(past).encode() if codec is JSON_PAYLOAD_CODEC else msgpack.packb(past, use_bin_type=True) with self.assertRaises(CultCachePayloadDepthError): - document.decode_payload(document.payload_codec.encode(past)) + document.decode_payload(raw) + if codec is None: + with self.assertRaises(PayloadCodecFault) as refused: + MESSAGEPACK_PAYLOAD_CODEC.encode(past) + self.assertEqual(refused.exception.code, "depth_limit") def test_a_tuple_and_a_container_dict_key_count_as_containers(self) -> None: document = define_document_type("tests.tuple_tree", encode=lambda value: value, payload_codec=_STUB_CODEC) diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 29984b1f..21892f11 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -3,9 +3,8 @@ from __future__ import annotations from dataclasses import dataclass from typing import Any, Iterable -import msgpack from cultcache_py import CultCache, CultCacheEnvelope -from cultcache_py.documents import DocumentDefinition +from cultcache_py.documents import MESSAGEPACK_PAYLOAD_CODEC, DocumentDefinition from .shard_log import CultNetShardLogResponse from .snapshot import CultNetRawDocumentRecord, CultNetRawSnapshotResponse @@ -187,9 +186,9 @@ def resolve_document_and_schema_id_for_raw_record( def _infer_schema_version_from_payload(payload: bytes) -> str | None: """Routing only, as the C# `CultNetDocumentRegistry.TryResolveDescriptorByPayloadSchema` does: reads a schema stamp off a raw record whose schema id is unknown, to pick a definition. It decides no format; the definition it picks - has a codec that decodes the payload or refuses it.""" + has a codec that decodes the payload or refuses it. The codec's reader reads the stamp, so the router decides no key.""" try: - decoded = msgpack.unpackb(payload, raw=False) + decoded = MESSAGEPACK_PAYLOAD_CODEC.decode(payload) except Exception: return None if isinstance(decoded, list) and decoded and isinstance(decoded[0], str): diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 5b78db1c..373b42cc 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -62,6 +62,7 @@ from cultnet_py import ( TcpFramedTransportConnection, CultNetWitnessArtifactBundle, apply_raw_document_record, + resolve_document_and_schema_id_for_raw_record, apply_raw_snapshot, apply_shard_log_response, create_rudp_transport_profile, @@ -2002,6 +2003,37 @@ class CultNetTests(unittest.TestCase): self.assertEqual([change.change_kind for change in typed_applied_log], ["updated", "removed"]) self.assertIsNone(cache.get(document, "item:1")) + def test_the_stamp_router_judges_no_map_key_and_the_codec_alone_does(self) -> None: + import msgpack # type: ignore + + document = define_database_entry_type( + "replica.stamped", + [("schema_version", 0), ("value", 1)], + schema_id="replica.stamped.current", + schema_name="replica.stamped", + schema_version="replica.stamped.v1", + ) + by_schema_id = {"replica.stamped.current": document, "replica.stamped": document} + + def routed(tree: object) -> object: + record = {"payload": msgpack.packb(tree, use_bin_type=True)} + return resolve_document_and_schema_id_for_raw_record(by_schema_id, "sha256:unknown", record) + + for name, tree in ( + ("array", ["replica.stamped.v1", {1: "a"}]), + ("map", {"schemaVersion": "replica.stamped.v1", "v": {1: "a"}}), + ("nested", ["replica.stamped.v1", {"a": {2: [{3: 4}]}}]), + ): + with self.subTest(form=name): + self.assertEqual(routed(tree), (document, "replica.stamped.current")) + for name, tree in ( + ("array", ["replica.stamped.v1", {None: "a"}]), + ("map", {"schemaVersion": "replica.stamped.v1", "v": {None: "a"}}), + ): + with self.subTest(form=name): + with self.assertRaises(KeyError): + routed(tree) + def test_cultnet_replication_helpers_recover_schema_stamped_raw_records(self) -> None: document = define_database_entry_type( "replica.runtime-policy",