cultcache-py: one container walk owns payload depth, float32 key images are refused on write, one reader owns map keys

_containers is the only depth counter (codec write, definition read and write); _PACK_DEPTH_LIMIT is gone. _check_writable refuses keys that collide once decoded (a Float32 beside an equal int, bool or float). The store's whole-file read and cultnet-py's stamp router read through the codec's reader instead of strict msgpack.unpackb, so no reader outside the codec decides key acceptability.

Does not pass the suite alone: two tests in test_payload_codec.py still assert the 511 bound; the next commit replaces them.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-12 07:40:02 +02:00
commit fe93b66cfb
5 changed files with 98 additions and 35 deletions

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import struct
from dataclasses import MISSING, dataclass, field
from collections.abc import Mapping, Sequence, Set
from itertools import chain
@ -44,7 +45,7 @@ class CultCachePayloadDepthError(ValueError):
class PayloadCodecFault(ValueError):
"""A codec raises this for a payload its format cannot carry. `code` names the rule (the library's are
`map_key_kind`, `duplicate_map_key` and `integer_out_of_range`) and `kind` the kind of node at fault (a MessagePack
`map_key_kind`, `duplicate_map_key`, `integer_out_of_range` and `depth_limit`) and `kind` the kind of node at fault (a MessagePack
kind such as `nil` or `bin`); neither is ever a payload value. A consumer-built codec may raise it."""
def __init__(self, code: str, kind: str = "") -> None:
@ -71,7 +72,7 @@ class CultCachePayloadFormatError(ValueError):
f"{codec} payloads take str, int, float, bool or timestamp keys (refused on {direction})."
)
elif code == "duplicate_map_key":
message = f"{type_name} payload has two map keys Python holds as one; refused on read."
message = f"{type_name} payload has two map keys Python holds as one; refused on {direction}."
elif code == "integer_out_of_range":
message = f"{type_name} payload has an integer beyond 64 bits; refused on write."
else:
@ -79,20 +80,38 @@ class CultCachePayloadFormatError(ValueError):
super().__init__(message)
def _check_payload_tree(tree: Any, type_name: str, direction: str) -> None:
"""The one depth walk over a payload tree, for every codec. Iterative (every mapping, sequence and set that is not
text or bytes is a container): refuses the first container past `MAX_PAYLOAD_DEPTH`."""
def _is_container(node: Any) -> bool:
return isinstance(node, _CONTAINERS) and not isinstance(node, _SCALAR_SEQUENCES)
def _children(node: Any) -> Iterable[Any]:
return chain(node.keys(), node.values()) if isinstance(node, Mapping) else node
def _containers(tree: Any) -> Iterable[Any]:
"""The one depth walk over a payload tree, for every codec and every caller. Iterative, so a cycle ends at the limit:
yields every container (a mapping, sequence or set that is not text or bytes; a mapping's keys and values are both
children), the outermost at depth 1, and raises `PayloadCodecFault('depth_limit')` at the first one past
`MAX_PAYLOAD_DEPTH`."""
stack = [(tree, 1)] if _is_container(tree) else []
while stack:
node, depth = stack.pop()
if depth > MAX_PAYLOAD_DEPTH:
raise CultCachePayloadDepthError(type_name, direction)
children = chain(node.keys(), node.values()) if isinstance(node, Mapping) else node
stack.extend((child, depth + 1) for child in children if _is_container(child))
raise PayloadCodecFault("depth_limit")
yield node
stack.extend((child, depth + 1) for child in _children(node) if _is_container(child))
def _is_container(node: Any) -> bool:
return isinstance(node, _CONTAINERS) and not isinstance(node, _SCALAR_SEQUENCES)
def _check_payload_tree(tree: Any, type_name: str, direction: str) -> None:
"""The definition's translation of the depth walk into its typed refusal."""
try:
for _ in _containers(tree):
pass
except PayloadCodecFault:
refusal = CultCachePayloadDepthError(type_name, direction)
else:
return
raise refusal
def _default_encode(value: Any) -> Any:
@ -139,8 +158,8 @@ def _pack_tree(tree: Any) -> bytes:
@dataclass(frozen=True)
class PayloadCodec:
"""A codec owns one payload format: how a tree becomes bytes and back, which of its decode faults mean the payload
nests past the interpreter's own stack, and which trees it can carry: it raises `PayloadCodecFault` for a key or an
integer its format cannot hold, and nothing it writes is refused by its own `decode`. A consumer-built codec owns
nests past the interpreter's own stack, and which trees it can carry: it raises `PayloadCodecFault` for a key, an
integer or a nesting its format cannot hold, and nothing it writes is refused by its own `decode`. A consumer-built codec owns
the depth of nodes the depth walk cannot see (anything that is not a Mapping, Sequence or Set)."""
name: str
@ -150,7 +169,6 @@ class PayloadCodec:
_INTEGER_RANGE = (-(2**63), 2**64 - 1)
_PACK_DEPTH_LIMIT = 511 # msgpack's own packb limit: a tree nested past it, or a cycle, is not walked on
_KEY_TYPES = (str, int, float, Float32, msgpack.Timestamp)
@ -181,24 +199,27 @@ def _checked_map(pairs: list[tuple[Any, Any]]) -> dict[Any, Any]:
return held
def _key_image(key: Any) -> Any:
"""A map key as the reader holds it: a `Float32` is written as float32 and read back as that float."""
return struct.unpack(">f", struct.pack(">f", key.value))[0] if isinstance(key, Float32) else key
def _check_writable(tree: Any) -> None:
"""The write side of the key and integer rule: what `_checked_map` refuses on read is refused here, so nothing
this codec writes is refused by its own decode. Iterative; the nesting limit is the definition's rule (a tree past it
is the definition's refusal), so the walk stops only where `packb` would, which also ends a cycle."""
stack = [(tree, 1)]
while stack:
node, depth = stack.pop()
if depth > _PACK_DEPTH_LIMIT:
raise ValueError("recursion limit exceeded")
if isinstance(node, int) and not _INTEGER_RANGE[0] <= node <= _INTEGER_RANGE[1]:
raise PayloadCodecFault("integer_out_of_range")
"""The write side of the key, integer and depth rule: what `_checked_map` refuses on read, and nesting past
`MAX_PAYLOAD_DEPTH`, is refused here, so nothing this codec writes is refused by its own decode. Keys that collide
once decoded (a `Float32` beside an equal int, bool or float) are the reader's duplicate."""
if isinstance(tree, int) and not _INTEGER_RANGE[0] <= tree <= _INTEGER_RANGE[1]:
raise PayloadCodecFault("integer_out_of_range")
for node in _containers(tree):
for child in _children(node):
if isinstance(child, int) and not _INTEGER_RANGE[0] <= child <= _INTEGER_RANGE[1]:
raise PayloadCodecFault("integer_out_of_range")
if isinstance(node, Mapping):
for key in node:
if not isinstance(key, _KEY_TYPES):
raise PayloadCodecFault("map_key_kind", _node_kind(key))
stack.extend((child, depth + 1) for child in chain(node.keys(), node.values()))
elif isinstance(node, (list, tuple)):
stack.extend((child, depth + 1) for child in node)
if len(dict.fromkeys(_key_image(key) for key in node)) != len(node):
raise PayloadCodecFault("duplicate_map_key")
def _unpack_tree(payload: bytes) -> Any:
@ -222,8 +243,8 @@ MESSAGEPACK_PAYLOAD_CODEC = PayloadCodec(
"MessagePack", _pack_checked_tree, _unpack_tree, depth_faults=(msgpack.exceptions.StackError,)
)
"""The default payload format. It carries what the C# reference's object reader accepts: str, int, float, bool and
timestamp keys, no duplicate key, and integers within -(2**63)..2**64-1. It refuses the rest on read through its map
hook and on write through its encode, so it never writes a record it cannot read."""
timestamp keys, no duplicate key, and integers within -(2**63)..2**64-1, nested no deeper than `MAX_PAYLOAD_DEPTH`. It refuses the rest on read through its
map hook and on write through its encode, so it never writes a record it cannot read."""
JSON_PAYLOAD_CODEC = PayloadCodec("JSON", _dump_json, _load_json, depth_faults=(RecursionError,))
"""The 0.3 default, kept for stores written before MessagePack: it writes exactly the 0.3 bytes. Never a default."""
@ -263,7 +284,9 @@ class UnionArm:
value: Any = None
def _fault_refusal(type_name: str, codec: PayloadCodec, direction: str, fault: PayloadCodecFault) -> CultCachePayloadFormatError:
def _fault_refusal(type_name: str, codec: PayloadCodec, direction: str, fault: PayloadCodecFault) -> CultCachePayloadFormatError | CultCachePayloadDepthError:
if fault.code == "depth_limit":
return CultCachePayloadDepthError(type_name, direction)
return CultCachePayloadFormatError(type_name, codec.name, direction, fault.code, fault.kind)

View file

@ -14,6 +14,7 @@ from .backing_store import (
CultCacheSchemaCatalogEntry,
CultCacheSchemaCatalogMember,
)
from .documents import _unpack_tree
STORE_FORMAT_VERSION = "cultcache.store.v1"
_STORE_FORMAT_PREFIX = "cultcache.store."
@ -95,7 +96,7 @@ class SingleFileMessagePackBackingStore:
data = _read_store(self.path, Path.read_bytes)
if data is None:
return []
decoded = msgpack.unpackb(data, raw=False)
decoded = _unpack_tree(data)
snapshot = _decode_v1_snapshot(decoded)
if snapshot is not None:
return snapshot

View file

@ -2,6 +2,7 @@
import collections
import dataclasses
import json
import shutil
import tempfile
import traceback
@ -16,6 +17,8 @@ from cultcache_py import (
MAX_SLOT_KEY,
CultCache,
JSON_PAYLOAD_CODEC,
MESSAGEPACK_PAYLOAD_CODEC,
PayloadCodecFault,
CultCachePayloadDepthError,
DatabaseEntryField,
PayloadCodec,
@ -645,8 +648,13 @@ class PayloadDepthRuleTests(unittest.TestCase):
past = _nest(257, mixed)
with self.assertRaises(CultCachePayloadDepthError):
document.encode_payload(past)
raw = json.dumps(past).encode() if codec is JSON_PAYLOAD_CODEC else msgpack.packb(past, use_bin_type=True)
with self.assertRaises(CultCachePayloadDepthError):
document.decode_payload(document.payload_codec.encode(past))
document.decode_payload(raw)
if codec is None:
with self.assertRaises(PayloadCodecFault) as refused:
MESSAGEPACK_PAYLOAD_CODEC.encode(past)
self.assertEqual(refused.exception.code, "depth_limit")
def test_a_tuple_and_a_container_dict_key_count_as_containers(self) -> None:
document = define_document_type("tests.tuple_tree", encode=lambda value: value, payload_codec=_STUB_CODEC)

View file

@ -3,9 +3,8 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Iterable
import msgpack
from cultcache_py import CultCache, CultCacheEnvelope
from cultcache_py.documents import DocumentDefinition
from cultcache_py.documents import MESSAGEPACK_PAYLOAD_CODEC, DocumentDefinition
from .shard_log import CultNetShardLogResponse
from .snapshot import CultNetRawDocumentRecord, CultNetRawSnapshotResponse
@ -187,9 +186,9 @@ def resolve_document_and_schema_id_for_raw_record(
def _infer_schema_version_from_payload(payload: bytes) -> str | None:
"""Routing only, as the C# `CultNetDocumentRegistry.TryResolveDescriptorByPayloadSchema` does: reads a schema stamp
off a raw record whose schema id is unknown, to pick a definition. It decides no format; the definition it picks
has a codec that decodes the payload or refuses it."""
has a codec that decodes the payload or refuses it. The codec's reader reads the stamp, so the router decides no key."""
try:
decoded = msgpack.unpackb(payload, raw=False)
decoded = MESSAGEPACK_PAYLOAD_CODEC.decode(payload)
except Exception:
return None
if isinstance(decoded, list) and decoded and isinstance(decoded[0], str):

View file

@ -62,6 +62,7 @@ from cultnet_py import (
TcpFramedTransportConnection,
CultNetWitnessArtifactBundle,
apply_raw_document_record,
resolve_document_and_schema_id_for_raw_record,
apply_raw_snapshot,
apply_shard_log_response,
create_rudp_transport_profile,
@ -2002,6 +2003,37 @@ class CultNetTests(unittest.TestCase):
self.assertEqual([change.change_kind for change in typed_applied_log], ["updated", "removed"])
self.assertIsNone(cache.get(document, "item:1"))
def test_the_stamp_router_judges_no_map_key_and_the_codec_alone_does(self) -> None:
import msgpack # type: ignore
document = define_database_entry_type(
"replica.stamped",
[("schema_version", 0), ("value", 1)],
schema_id="replica.stamped.current",
schema_name="replica.stamped",
schema_version="replica.stamped.v1",
)
by_schema_id = {"replica.stamped.current": document, "replica.stamped": document}
def routed(tree: object) -> object:
record = {"payload": msgpack.packb(tree, use_bin_type=True)}
return resolve_document_and_schema_id_for_raw_record(by_schema_id, "sha256:unknown", record)
for name, tree in (
("array", ["replica.stamped.v1", {1: "a"}]),
("map", {"schemaVersion": "replica.stamped.v1", "v": {1: "a"}}),
("nested", ["replica.stamped.v1", {"a": {2: [{3: 4}]}}]),
):
with self.subTest(form=name):
self.assertEqual(routed(tree), (document, "replica.stamped.current"))
for name, tree in (
("array", ["replica.stamped.v1", {None: "a"}]),
("map", {"schemaVersion": "replica.stamped.v1", "v": {None: "a"}}),
):
with self.subTest(form=name):
with self.assertRaises(KeyError):
routed(tree)
def test_cultnet_replication_helpers_recover_schema_stamped_raw_records(self) -> None:
document = define_database_entry_type(
"replica.runtime-policy",